($INBOX_DIR/description missing)
help / color / mirror / Atom feed[PATCH v8 3/8] f! progress reporting..
73+ messages / 4 participants
[nested] [flat]
* [PATCH v9 3/8] f! progress reporting..
@ 2021-02-09 02:45 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 73+ messages in thread
From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)
This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.
This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
doc/src/sgml/monitoring.sgml | 21 +++++++++++
src/backend/catalog/system_views.sql | 4 +-
src/backend/commands/cluster.c | 55 ++++++++++++++++------------
src/backend/commands/vacuum.c | 5 +++
src/include/commands/progress.h | 3 ++
src/test/regress/expected/rules.out | 4 +-
6 files changed, 67 insertions(+), 25 deletions(-)
diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml
index 3513e127b7..dca47f6a48 100644
--- a/doc/src/sgml/monitoring.sgml
+++ b/doc/src/sgml/monitoring.sgml
@@ -6231,6 +6231,27 @@ SELECT pg_stat_get_backend_pid(s.backendid) AS pid,
is <literal>rebuilding index</literal>.
</para></entry>
</row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>total_partitions</structfield> <type>bigint</type>
+ </para>
+ When clustering a partitioned table, this column is set to the total
+ number of partitions to be clustered.
+ <para>
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>done_partitions</structfield> <type>bigint</type>
+ </para>
+ <para>
+ When clustering a partitioned table, this column is set to the number
+ of partitions which have been clustered.
+ </para></entry>
+ </row>
+
</tbody>
</tgroup>
</table>
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fc94a73a54..21c27f0437 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1070,7 +1070,9 @@ CREATE VIEW pg_stat_progress_cluster AS
S.param5 AS heap_tuples_written,
S.param6 AS heap_blks_total,
S.param7 AS heap_blks_scanned,
- S.param8 AS index_rebuild_count
+ S.param8 AS index_rebuild_count,
+ S.param18 AS partitions_total,
+ S.param19 AS partitions_done
FROM pg_stat_get_progress_info('CLUSTER') AS S
LEFT JOIN pg_database D ON S.datid = D.oid;
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 0c08ac56dc..7d8d1d8a69 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
static List *get_tables_to_cluster(MemoryContext cluster_context);
static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
/*---------------------------------------------------------------------------
@@ -188,16 +188,26 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
/* close relation, keep lock till commit */
table_close(rel, ShareUpdateExclusiveLock);
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
{
/* Do the job. */
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_CLUSTER);
cluster_rel(tableOid, indexOid, ¶ms);
+ pgstat_progress_end_command();
}
else
{
List *rvs;
MemoryContext cluster_context;
+ int64 progress_val[2];
+ const int progress_index[2] = {
+ PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+ };
+
/* Refuse to hold strong locks in a user transaction */
PreventInTransactionBlock(isTopLevel, "CLUSTER");
@@ -206,7 +216,15 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
ALLOCSET_DEFAULT_SIZES);
rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
- cluster_multiple_rels(rvs, params.options);
+
+ /* Report command and number of partitions */
+ progress_val[0] = PROGRESS_CLUSTER_COMMAND_CLUSTER;
+ /* XXX: rvs includes the parent, which is not a "partition" */
+ progress_val[1] = list_length(rvs);
+
+ pgstat_progress_update_multi_param(2, progress_index, progress_val);
+ cluster_multiple_rels(rvs, params.options, true);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -245,7 +263,11 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
* cluster_context.
*/
rvs = get_tables_to_cluster(cluster_context);
- cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_CLUSTER);
+ cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -282,14 +304,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* Check for user-requested abort. */
CHECK_FOR_INTERRUPTS();
- pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
- if (OidIsValid(indexOid))
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_CLUSTER);
- else
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
/*
* We grab exclusive access to the target rel and index for the duration
* of the transaction. (This is redundant for the single-transaction
@@ -300,10 +314,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* If the table has gone away, we can skip processing it */
if (!OldHeap)
- {
- pgstat_progress_end_command();
return;
- }
/*
* Since we may open a new transaction for each relation, we have to check
@@ -319,7 +330,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!pg_class_ownercheck(tableOid, GetUserId()))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -334,7 +344,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (RELATION_IS_OTHER_TEMP(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -346,7 +355,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -357,7 +365,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!get_index_isclustered(indexOid))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
}
@@ -416,7 +423,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!RelationIsPopulated(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -424,7 +430,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -440,8 +445,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
rebuild_relation(OldHeap, indexOid, verbose);
/* NB: rebuild_relation does table_close() on OldHeap */
-
- pgstat_progress_end_command();
}
/*
@@ -1632,9 +1635,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
/* Cluster each relation in a separate transaction */
static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
{
ListCell *lc;
+ off_t childnum = 0;
/* Commit to get out of starting transaction */
PopActiveSnapshot();
@@ -1657,6 +1661,11 @@ cluster_multiple_rels(List *rvs, int options)
cluster_rel(rvtc->tableOid, rvtc->indexOid,
&cluster_params);
+ if (ispartitioned)
+ pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+ ++childnum);
+
+
PopActiveSnapshot();
CommitTransactionCommand();
}
diff --git a/src/backend/commands/vacuum.c b/src/backend/commands/vacuum.c
index c064352e23..05d52858b1 100644
--- a/src/backend/commands/vacuum.c
+++ b/src/backend/commands/vacuum.c
@@ -37,6 +37,7 @@
#include "catalog/pg_namespace.h"
#include "commands/cluster.h"
#include "commands/defrem.h"
+#include "commands/progress.h"
#include "commands/vacuum.h"
#include "miscadmin.h"
#include "nodes/makefuncs.h"
@@ -1937,7 +1938,11 @@ vacuum_rel(Oid relid, RangeVar *relation, VacuumParams *params)
cluster_params.options |= CLUOPT_VERBOSE;
/* VACUUM FULL is now a variant of CLUSTER; see cluster.c */
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, relid);
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
cluster_rel(relid, InvalidOid, &cluster_params);
+ pgstat_progress_end_command();
}
else
table_relation_vacuum(onerel, params, vac_strategy);
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
#define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS 5
#define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED 6
#define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT 7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL 17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE 18
/* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
#define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP 1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b1c9b7bdfe..c888415235 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1943,7 +1943,9 @@ pg_stat_progress_cluster| SELECT s.pid,
s.param5 AS heap_tuples_written,
s.param6 AS heap_blks_total,
s.param7 AS heap_blks_scanned,
- s.param8 AS index_rebuild_count
+ s.param8 AS index_rebuild_count,
+ s.param18 AS partitions_total,
+ s.param19 AS partitions_done
FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
LEFT JOIN pg_database d ON ((s.datid = d.oid)));
pg_stat_progress_copy| SELECT s.pid,
--
2.17.0
--fmvA4kSBHQVZhkR6
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v9-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"
^ permalink raw reply [nested|flat] 73+ messages in thread
* [PATCH v9 3/8] f! progress reporting..
@ 2021-02-09 02:45 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 73+ messages in thread
From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)
This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.
This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
doc/src/sgml/monitoring.sgml | 21 +++++++++++
src/backend/catalog/system_views.sql | 4 +-
src/backend/commands/cluster.c | 55 ++++++++++++++++------------
src/backend/commands/vacuum.c | 5 +++
src/include/commands/progress.h | 3 ++
src/test/regress/expected/rules.out | 4 +-
6 files changed, 67 insertions(+), 25 deletions(-)
diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml
index 3513e127b7..dca47f6a48 100644
--- a/doc/src/sgml/monitoring.sgml
+++ b/doc/src/sgml/monitoring.sgml
@@ -6231,6 +6231,27 @@ SELECT pg_stat_get_backend_pid(s.backendid) AS pid,
is <literal>rebuilding index</literal>.
</para></entry>
</row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>total_partitions</structfield> <type>bigint</type>
+ </para>
+ When clustering a partitioned table, this column is set to the total
+ number of partitions to be clustered.
+ <para>
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>done_partitions</structfield> <type>bigint</type>
+ </para>
+ <para>
+ When clustering a partitioned table, this column is set to the number
+ of partitions which have been clustered.
+ </para></entry>
+ </row>
+
</tbody>
</tgroup>
</table>
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fc94a73a54..21c27f0437 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1070,7 +1070,9 @@ CREATE VIEW pg_stat_progress_cluster AS
S.param5 AS heap_tuples_written,
S.param6 AS heap_blks_total,
S.param7 AS heap_blks_scanned,
- S.param8 AS index_rebuild_count
+ S.param8 AS index_rebuild_count,
+ S.param18 AS partitions_total,
+ S.param19 AS partitions_done
FROM pg_stat_get_progress_info('CLUSTER') AS S
LEFT JOIN pg_database D ON S.datid = D.oid;
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 0c08ac56dc..7d8d1d8a69 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
static List *get_tables_to_cluster(MemoryContext cluster_context);
static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
/*---------------------------------------------------------------------------
@@ -188,16 +188,26 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
/* close relation, keep lock till commit */
table_close(rel, ShareUpdateExclusiveLock);
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
{
/* Do the job. */
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_CLUSTER);
cluster_rel(tableOid, indexOid, ¶ms);
+ pgstat_progress_end_command();
}
else
{
List *rvs;
MemoryContext cluster_context;
+ int64 progress_val[2];
+ const int progress_index[2] = {
+ PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+ };
+
/* Refuse to hold strong locks in a user transaction */
PreventInTransactionBlock(isTopLevel, "CLUSTER");
@@ -206,7 +216,15 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
ALLOCSET_DEFAULT_SIZES);
rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
- cluster_multiple_rels(rvs, params.options);
+
+ /* Report command and number of partitions */
+ progress_val[0] = PROGRESS_CLUSTER_COMMAND_CLUSTER;
+ /* XXX: rvs includes the parent, which is not a "partition" */
+ progress_val[1] = list_length(rvs);
+
+ pgstat_progress_update_multi_param(2, progress_index, progress_val);
+ cluster_multiple_rels(rvs, params.options, true);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -245,7 +263,11 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
* cluster_context.
*/
rvs = get_tables_to_cluster(cluster_context);
- cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_CLUSTER);
+ cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -282,14 +304,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* Check for user-requested abort. */
CHECK_FOR_INTERRUPTS();
- pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
- if (OidIsValid(indexOid))
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_CLUSTER);
- else
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
/*
* We grab exclusive access to the target rel and index for the duration
* of the transaction. (This is redundant for the single-transaction
@@ -300,10 +314,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* If the table has gone away, we can skip processing it */
if (!OldHeap)
- {
- pgstat_progress_end_command();
return;
- }
/*
* Since we may open a new transaction for each relation, we have to check
@@ -319,7 +330,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!pg_class_ownercheck(tableOid, GetUserId()))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -334,7 +344,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (RELATION_IS_OTHER_TEMP(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -346,7 +355,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -357,7 +365,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!get_index_isclustered(indexOid))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
}
@@ -416,7 +423,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!RelationIsPopulated(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -424,7 +430,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -440,8 +445,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
rebuild_relation(OldHeap, indexOid, verbose);
/* NB: rebuild_relation does table_close() on OldHeap */
-
- pgstat_progress_end_command();
}
/*
@@ -1632,9 +1635,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
/* Cluster each relation in a separate transaction */
static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
{
ListCell *lc;
+ off_t childnum = 0;
/* Commit to get out of starting transaction */
PopActiveSnapshot();
@@ -1657,6 +1661,11 @@ cluster_multiple_rels(List *rvs, int options)
cluster_rel(rvtc->tableOid, rvtc->indexOid,
&cluster_params);
+ if (ispartitioned)
+ pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+ ++childnum);
+
+
PopActiveSnapshot();
CommitTransactionCommand();
}
diff --git a/src/backend/commands/vacuum.c b/src/backend/commands/vacuum.c
index c064352e23..05d52858b1 100644
--- a/src/backend/commands/vacuum.c
+++ b/src/backend/commands/vacuum.c
@@ -37,6 +37,7 @@
#include "catalog/pg_namespace.h"
#include "commands/cluster.h"
#include "commands/defrem.h"
+#include "commands/progress.h"
#include "commands/vacuum.h"
#include "miscadmin.h"
#include "nodes/makefuncs.h"
@@ -1937,7 +1938,11 @@ vacuum_rel(Oid relid, RangeVar *relation, VacuumParams *params)
cluster_params.options |= CLUOPT_VERBOSE;
/* VACUUM FULL is now a variant of CLUSTER; see cluster.c */
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, relid);
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
cluster_rel(relid, InvalidOid, &cluster_params);
+ pgstat_progress_end_command();
}
else
table_relation_vacuum(onerel, params, vac_strategy);
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
#define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS 5
#define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED 6
#define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT 7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL 17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE 18
/* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
#define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP 1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b1c9b7bdfe..c888415235 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1943,7 +1943,9 @@ pg_stat_progress_cluster| SELECT s.pid,
s.param5 AS heap_tuples_written,
s.param6 AS heap_blks_total,
s.param7 AS heap_blks_scanned,
- s.param8 AS index_rebuild_count
+ s.param8 AS index_rebuild_count,
+ s.param18 AS partitions_total,
+ s.param19 AS partitions_done
FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
LEFT JOIN pg_database d ON ((s.datid = d.oid)));
pg_stat_progress_copy| SELECT s.pid,
--
2.17.0
--fmvA4kSBHQVZhkR6
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v9-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"
^ permalink raw reply [nested|flat] 73+ messages in thread
* [PATCH v9 3/8] f! progress reporting..
@ 2021-02-09 02:45 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 73+ messages in thread
From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)
This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.
This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
doc/src/sgml/monitoring.sgml | 21 +++++++++++
src/backend/catalog/system_views.sql | 4 +-
src/backend/commands/cluster.c | 55 ++++++++++++++++------------
src/backend/commands/vacuum.c | 5 +++
src/include/commands/progress.h | 3 ++
src/test/regress/expected/rules.out | 4 +-
6 files changed, 67 insertions(+), 25 deletions(-)
diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml
index 3513e127b7..dca47f6a48 100644
--- a/doc/src/sgml/monitoring.sgml
+++ b/doc/src/sgml/monitoring.sgml
@@ -6231,6 +6231,27 @@ SELECT pg_stat_get_backend_pid(s.backendid) AS pid,
is <literal>rebuilding index</literal>.
</para></entry>
</row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>total_partitions</structfield> <type>bigint</type>
+ </para>
+ When clustering a partitioned table, this column is set to the total
+ number of partitions to be clustered.
+ <para>
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>done_partitions</structfield> <type>bigint</type>
+ </para>
+ <para>
+ When clustering a partitioned table, this column is set to the number
+ of partitions which have been clustered.
+ </para></entry>
+ </row>
+
</tbody>
</tgroup>
</table>
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fc94a73a54..21c27f0437 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1070,7 +1070,9 @@ CREATE VIEW pg_stat_progress_cluster AS
S.param5 AS heap_tuples_written,
S.param6 AS heap_blks_total,
S.param7 AS heap_blks_scanned,
- S.param8 AS index_rebuild_count
+ S.param8 AS index_rebuild_count,
+ S.param18 AS partitions_total,
+ S.param19 AS partitions_done
FROM pg_stat_get_progress_info('CLUSTER') AS S
LEFT JOIN pg_database D ON S.datid = D.oid;
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 0c08ac56dc..7d8d1d8a69 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
static List *get_tables_to_cluster(MemoryContext cluster_context);
static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
/*---------------------------------------------------------------------------
@@ -188,16 +188,26 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
/* close relation, keep lock till commit */
table_close(rel, ShareUpdateExclusiveLock);
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
{
/* Do the job. */
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_CLUSTER);
cluster_rel(tableOid, indexOid, ¶ms);
+ pgstat_progress_end_command();
}
else
{
List *rvs;
MemoryContext cluster_context;
+ int64 progress_val[2];
+ const int progress_index[2] = {
+ PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+ };
+
/* Refuse to hold strong locks in a user transaction */
PreventInTransactionBlock(isTopLevel, "CLUSTER");
@@ -206,7 +216,15 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
ALLOCSET_DEFAULT_SIZES);
rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
- cluster_multiple_rels(rvs, params.options);
+
+ /* Report command and number of partitions */
+ progress_val[0] = PROGRESS_CLUSTER_COMMAND_CLUSTER;
+ /* XXX: rvs includes the parent, which is not a "partition" */
+ progress_val[1] = list_length(rvs);
+
+ pgstat_progress_update_multi_param(2, progress_index, progress_val);
+ cluster_multiple_rels(rvs, params.options, true);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -245,7 +263,11 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
* cluster_context.
*/
rvs = get_tables_to_cluster(cluster_context);
- cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_CLUSTER);
+ cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -282,14 +304,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* Check for user-requested abort. */
CHECK_FOR_INTERRUPTS();
- pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
- if (OidIsValid(indexOid))
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_CLUSTER);
- else
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
/*
* We grab exclusive access to the target rel and index for the duration
* of the transaction. (This is redundant for the single-transaction
@@ -300,10 +314,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* If the table has gone away, we can skip processing it */
if (!OldHeap)
- {
- pgstat_progress_end_command();
return;
- }
/*
* Since we may open a new transaction for each relation, we have to check
@@ -319,7 +330,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!pg_class_ownercheck(tableOid, GetUserId()))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -334,7 +344,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (RELATION_IS_OTHER_TEMP(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -346,7 +355,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -357,7 +365,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!get_index_isclustered(indexOid))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
}
@@ -416,7 +423,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!RelationIsPopulated(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -424,7 +430,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -440,8 +445,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
rebuild_relation(OldHeap, indexOid, verbose);
/* NB: rebuild_relation does table_close() on OldHeap */
-
- pgstat_progress_end_command();
}
/*
@@ -1632,9 +1635,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
/* Cluster each relation in a separate transaction */
static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
{
ListCell *lc;
+ off_t childnum = 0;
/* Commit to get out of starting transaction */
PopActiveSnapshot();
@@ -1657,6 +1661,11 @@ cluster_multiple_rels(List *rvs, int options)
cluster_rel(rvtc->tableOid, rvtc->indexOid,
&cluster_params);
+ if (ispartitioned)
+ pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+ ++childnum);
+
+
PopActiveSnapshot();
CommitTransactionCommand();
}
diff --git a/src/backend/commands/vacuum.c b/src/backend/commands/vacuum.c
index c064352e23..05d52858b1 100644
--- a/src/backend/commands/vacuum.c
+++ b/src/backend/commands/vacuum.c
@@ -37,6 +37,7 @@
#include "catalog/pg_namespace.h"
#include "commands/cluster.h"
#include "commands/defrem.h"
+#include "commands/progress.h"
#include "commands/vacuum.h"
#include "miscadmin.h"
#include "nodes/makefuncs.h"
@@ -1937,7 +1938,11 @@ vacuum_rel(Oid relid, RangeVar *relation, VacuumParams *params)
cluster_params.options |= CLUOPT_VERBOSE;
/* VACUUM FULL is now a variant of CLUSTER; see cluster.c */
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, relid);
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
cluster_rel(relid, InvalidOid, &cluster_params);
+ pgstat_progress_end_command();
}
else
table_relation_vacuum(onerel, params, vac_strategy);
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
#define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS 5
#define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED 6
#define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT 7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL 17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE 18
/* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
#define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP 1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b1c9b7bdfe..c888415235 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1943,7 +1943,9 @@ pg_stat_progress_cluster| SELECT s.pid,
s.param5 AS heap_tuples_written,
s.param6 AS heap_blks_total,
s.param7 AS heap_blks_scanned,
- s.param8 AS index_rebuild_count
+ s.param8 AS index_rebuild_count,
+ s.param18 AS partitions_total,
+ s.param19 AS partitions_done
FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
LEFT JOIN pg_database d ON ((s.datid = d.oid)));
pg_stat_progress_copy| SELECT s.pid,
--
2.17.0
--fmvA4kSBHQVZhkR6
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v9-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"
^ permalink raw reply [nested|flat] 73+ messages in thread
* [PATCH v10 3/8] f! progress reporting..
@ 2021-02-09 02:45 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 73+ messages in thread
From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)
This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.
This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
doc/src/sgml/monitoring.sgml | 21 +++++++++++
src/backend/catalog/system_views.sql | 4 +-
src/backend/commands/cluster.c | 55 ++++++++++++++++------------
src/backend/commands/vacuum.c | 5 +++
src/include/commands/progress.h | 3 ++
src/test/regress/expected/rules.out | 4 +-
6 files changed, 67 insertions(+), 25 deletions(-)
diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml
index 56018745c8..28493c7931 100644
--- a/doc/src/sgml/monitoring.sgml
+++ b/doc/src/sgml/monitoring.sgml
@@ -6293,6 +6293,27 @@ SELECT pg_stat_get_backend_pid(s.backendid) AS pid,
is <literal>rebuilding index</literal>.
</para></entry>
</row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>total_partitions</structfield> <type>bigint</type>
+ </para>
+ When clustering a partitioned table, this column is set to the total
+ number of partitions to be clustered.
+ <para>
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>done_partitions</structfield> <type>bigint</type>
+ </para>
+ <para>
+ When clustering a partitioned table, this column is set to the number
+ of partitions which have been clustered.
+ </para></entry>
+ </row>
+
</tbody>
</tgroup>
</table>
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index 5f2541d316..0256809f06 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1142,7 +1142,9 @@ CREATE VIEW pg_stat_progress_cluster AS
S.param5 AS heap_tuples_written,
S.param6 AS heap_blks_total,
S.param7 AS heap_blks_scanned,
- S.param8 AS index_rebuild_count
+ S.param8 AS index_rebuild_count,
+ S.param18 AS partitions_total,
+ S.param19 AS partitions_done
FROM pg_stat_get_progress_info('CLUSTER') AS S
LEFT JOIN pg_database D ON S.datid = D.oid;
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 0c08ac56dc..7d8d1d8a69 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
static List *get_tables_to_cluster(MemoryContext cluster_context);
static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
/*---------------------------------------------------------------------------
@@ -188,16 +188,26 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
/* close relation, keep lock till commit */
table_close(rel, ShareUpdateExclusiveLock);
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
{
/* Do the job. */
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_CLUSTER);
cluster_rel(tableOid, indexOid, ¶ms);
+ pgstat_progress_end_command();
}
else
{
List *rvs;
MemoryContext cluster_context;
+ int64 progress_val[2];
+ const int progress_index[2] = {
+ PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+ };
+
/* Refuse to hold strong locks in a user transaction */
PreventInTransactionBlock(isTopLevel, "CLUSTER");
@@ -206,7 +216,15 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
ALLOCSET_DEFAULT_SIZES);
rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
- cluster_multiple_rels(rvs, params.options);
+
+ /* Report command and number of partitions */
+ progress_val[0] = PROGRESS_CLUSTER_COMMAND_CLUSTER;
+ /* XXX: rvs includes the parent, which is not a "partition" */
+ progress_val[1] = list_length(rvs);
+
+ pgstat_progress_update_multi_param(2, progress_index, progress_val);
+ cluster_multiple_rels(rvs, params.options, true);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -245,7 +263,11 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
* cluster_context.
*/
rvs = get_tables_to_cluster(cluster_context);
- cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_CLUSTER);
+ cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -282,14 +304,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* Check for user-requested abort. */
CHECK_FOR_INTERRUPTS();
- pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
- if (OidIsValid(indexOid))
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_CLUSTER);
- else
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
/*
* We grab exclusive access to the target rel and index for the duration
* of the transaction. (This is redundant for the single-transaction
@@ -300,10 +314,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* If the table has gone away, we can skip processing it */
if (!OldHeap)
- {
- pgstat_progress_end_command();
return;
- }
/*
* Since we may open a new transaction for each relation, we have to check
@@ -319,7 +330,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!pg_class_ownercheck(tableOid, GetUserId()))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -334,7 +344,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (RELATION_IS_OTHER_TEMP(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -346,7 +355,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -357,7 +365,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!get_index_isclustered(indexOid))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
}
@@ -416,7 +423,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!RelationIsPopulated(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -424,7 +430,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -440,8 +445,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
rebuild_relation(OldHeap, indexOid, verbose);
/* NB: rebuild_relation does table_close() on OldHeap */
-
- pgstat_progress_end_command();
}
/*
@@ -1632,9 +1635,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
/* Cluster each relation in a separate transaction */
static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
{
ListCell *lc;
+ off_t childnum = 0;
/* Commit to get out of starting transaction */
PopActiveSnapshot();
@@ -1657,6 +1661,11 @@ cluster_multiple_rels(List *rvs, int options)
cluster_rel(rvtc->tableOid, rvtc->indexOid,
&cluster_params);
+ if (ispartitioned)
+ pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+ ++childnum);
+
+
PopActiveSnapshot();
CommitTransactionCommand();
}
diff --git a/src/backend/commands/vacuum.c b/src/backend/commands/vacuum.c
index 662aff04b4..1d8a7ca774 100644
--- a/src/backend/commands/vacuum.c
+++ b/src/backend/commands/vacuum.c
@@ -37,6 +37,7 @@
#include "catalog/pg_namespace.h"
#include "commands/cluster.h"
#include "commands/defrem.h"
+#include "commands/progress.h"
#include "commands/vacuum.h"
#include "miscadmin.h"
#include "nodes/makefuncs.h"
@@ -1937,7 +1938,11 @@ vacuum_rel(Oid relid, RangeVar *relation, VacuumParams *params)
cluster_params.options |= CLUOPT_VERBOSE;
/* VACUUM FULL is now a variant of CLUSTER; see cluster.c */
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, relid);
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
cluster_rel(relid, InvalidOid, &cluster_params);
+ pgstat_progress_end_command();
}
else
table_relation_vacuum(onerel, params, vac_strategy);
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index c6b139d57d..d85781ae0b 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
#define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS 5
#define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED 6
#define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT 7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL 17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE 18
/* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
#define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP 1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index 9b59a7b4a5..a8bcc599ac 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1943,7 +1943,9 @@ pg_stat_progress_cluster| SELECT s.pid,
s.param5 AS heap_tuples_written,
s.param6 AS heap_blks_total,
s.param7 AS heap_blks_scanned,
- s.param8 AS index_rebuild_count
+ s.param8 AS index_rebuild_count,
+ s.param18 AS partitions_total,
+ s.param19 AS partitions_done
FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
LEFT JOIN pg_database d ON ((s.datid = d.oid)));
pg_stat_progress_copy| SELECT s.pid,
--
2.17.0
--mvpLiMfbWzRoNl4x
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v10-0004-Propagate-changes-to-indisclustered-to-child-par.patch"
^ permalink raw reply [nested|flat] 73+ messages in thread
* [PATCH v8 3/8] f! progress reporting..
@ 2021-02-09 02:45 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 73+ messages in thread
From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)
This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.
This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
src/backend/catalog/system_views.sql | 4 +-
src/backend/commands/cluster.c | 64 +++++++++++++++++-----------
src/include/commands/progress.h | 3 ++
src/test/regress/expected/rules.out | 4 +-
4 files changed, 47 insertions(+), 28 deletions(-)
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fa58afd9d7..9a3cb5f935 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1069,7 +1069,9 @@ CREATE VIEW pg_stat_progress_cluster AS
S.param5 AS heap_tuples_written,
S.param6 AS heap_blks_total,
S.param7 AS heap_blks_scanned,
- S.param8 AS index_rebuild_count
+ S.param8 AS index_rebuild_count,
+ S.param18 AS partitions_total,
+ S.param19 AS partitions_done
FROM pg_stat_get_progress_info('CLUSTER') AS S
LEFT JOIN pg_database D ON S.datid = D.oid;
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 9b6673867c..cb4fc350c6 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
static List *get_tables_to_cluster(MemoryContext cluster_context);
static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
/*---------------------------------------------------------------------------
@@ -188,16 +188,28 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
/* close relation, keep lock till commit */
table_close(rel, ShareUpdateExclusiveLock);
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
{
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ OidIsValid(indexOid) ? PROGRESS_CLUSTER_COMMAND_CLUSTER :
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
+
/* Do the job. */
cluster_rel(tableOid, indexOid, ¶ms);
+ pgstat_progress_end_command();
}
else
{
List *rvs;
MemoryContext cluster_context;
+ int64 progress_val[2];
+ const int progress_index[] = {
+ PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+ };
+
/* Refuse to hold strong locks in a user transaction */
PreventInTransactionBlock(isTopLevel, "CLUSTER");
@@ -206,7 +218,20 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
ALLOCSET_DEFAULT_SIZES);
rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
- cluster_multiple_rels(rvs, params.options);
+
+ /* Report vacuum full or cluster and number of partitions */
+ progress_val[0] = OidIsValid(indexOid) ?
+ PROGRESS_CLUSTER_COMMAND_CLUSTER :
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL;
+ progress_val[1] = list_length(rvs);
+// This is currently showing the total number of *tables*, including the
+// parent, and intermediate partitions, so the column should be renamed, or
+// we should count the number of leaf partitions
+
+ pgstat_progress_update_multi_param(2, progress_index, progress_val);
+
+ cluster_multiple_rels(rvs, params.options, true);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -245,7 +270,7 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
* cluster_context.
*/
rvs = get_tables_to_cluster(cluster_context);
- cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+ cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -282,14 +307,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* Check for user-requested abort. */
CHECK_FOR_INTERRUPTS();
- pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
- if (OidIsValid(indexOid))
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_CLUSTER);
- else
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
/*
* We grab exclusive access to the target rel and index for the duration
* of the transaction. (This is redundant for the single-transaction
@@ -300,10 +317,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* If the table has gone away, we can skip processing it */
if (!OldHeap)
- {
- pgstat_progress_end_command();
return;
- }
/*
* Since we may open a new transaction for each relation, we have to check
@@ -319,7 +333,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!pg_class_ownercheck(tableOid, GetUserId()))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -334,7 +347,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (RELATION_IS_OTHER_TEMP(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -346,7 +358,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -357,7 +368,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!get_index_isclustered(indexOid))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
}
@@ -416,7 +426,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!RelationIsPopulated(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -424,7 +433,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -440,8 +448,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
rebuild_relation(OldHeap, indexOid, verbose);
/* NB: rebuild_relation does table_close() on OldHeap */
-
- pgstat_progress_end_command();
}
/*
@@ -1366,7 +1372,7 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap,
ReindexParams reindex_params = {0};
int i;
- /* Report that we are now swapping relation files */
+ /* Report that we are now swapping relation files XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_SWAP_REL_FILES);
@@ -1417,13 +1423,13 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap,
else if (newrelpersistence == RELPERSISTENCE_PERMANENT)
reindex_flags |= REINDEX_REL_FORCE_INDEXES_PERMANENT;
- /* Report that we are now reindexing relations */
+ /* Report that we are now reindexing relations XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_REBUILD_INDEX);
reindex_relation(OIDOldHeap, reindex_flags, &reindex_params);
- /* Report that we are now doing clean up */
+ /* Report that we are now doing clean up XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_FINAL_CLEANUP);
@@ -1633,9 +1639,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
/* Cluster each relation in a separate transaction */
static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
{
ListCell *lc;
+ off_t childnum = 0;
/* Commit to get out of starting transaction */
PopActiveSnapshot();
@@ -1658,6 +1665,11 @@ cluster_multiple_rels(List *rvs, int options)
cluster_rel(rvtc->tableOid, rvtc->indexOid,
&cluster_params);
+ if (ispartitioned)
+ pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+ childnum++);
+
+
PopActiveSnapshot();
CommitTransactionCommand();
}
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
#define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS 5
#define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED 6
#define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT 7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL 17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE 18
/* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
#define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP 1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b632d9f2ea..d1fea532bc 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1941,7 +1941,9 @@ pg_stat_progress_cluster| SELECT s.pid,
s.param5 AS heap_tuples_written,
s.param6 AS heap_blks_total,
s.param7 AS heap_blks_scanned,
- s.param8 AS index_rebuild_count
+ s.param8 AS index_rebuild_count,
+ s.param18 AS partitions_total,
+ s.param19 AS partitions_done
FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
LEFT JOIN pg_database d ON ((s.datid = d.oid)));
pg_stat_progress_copy| SELECT s.pid,
--
2.17.0
--O5XBE6gyVG5Rl6Rj
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v8-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"
^ permalink raw reply [nested|flat] 73+ messages in thread
* [PATCH v8 3/8] f! progress reporting..
@ 2021-02-09 02:45 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 73+ messages in thread
From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)
This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.
This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
src/backend/catalog/system_views.sql | 4 +-
src/backend/commands/cluster.c | 64 +++++++++++++++++-----------
src/include/commands/progress.h | 3 ++
src/test/regress/expected/rules.out | 4 +-
4 files changed, 47 insertions(+), 28 deletions(-)
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fa58afd9d7..9a3cb5f935 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1069,7 +1069,9 @@ CREATE VIEW pg_stat_progress_cluster AS
S.param5 AS heap_tuples_written,
S.param6 AS heap_blks_total,
S.param7 AS heap_blks_scanned,
- S.param8 AS index_rebuild_count
+ S.param8 AS index_rebuild_count,
+ S.param18 AS partitions_total,
+ S.param19 AS partitions_done
FROM pg_stat_get_progress_info('CLUSTER') AS S
LEFT JOIN pg_database D ON S.datid = D.oid;
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 9b6673867c..cb4fc350c6 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
static List *get_tables_to_cluster(MemoryContext cluster_context);
static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
/*---------------------------------------------------------------------------
@@ -188,16 +188,28 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
/* close relation, keep lock till commit */
table_close(rel, ShareUpdateExclusiveLock);
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
{
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ OidIsValid(indexOid) ? PROGRESS_CLUSTER_COMMAND_CLUSTER :
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
+
/* Do the job. */
cluster_rel(tableOid, indexOid, ¶ms);
+ pgstat_progress_end_command();
}
else
{
List *rvs;
MemoryContext cluster_context;
+ int64 progress_val[2];
+ const int progress_index[] = {
+ PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+ };
+
/* Refuse to hold strong locks in a user transaction */
PreventInTransactionBlock(isTopLevel, "CLUSTER");
@@ -206,7 +218,20 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
ALLOCSET_DEFAULT_SIZES);
rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
- cluster_multiple_rels(rvs, params.options);
+
+ /* Report vacuum full or cluster and number of partitions */
+ progress_val[0] = OidIsValid(indexOid) ?
+ PROGRESS_CLUSTER_COMMAND_CLUSTER :
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL;
+ progress_val[1] = list_length(rvs);
+// This is currently showing the total number of *tables*, including the
+// parent, and intermediate partitions, so the column should be renamed, or
+// we should count the number of leaf partitions
+
+ pgstat_progress_update_multi_param(2, progress_index, progress_val);
+
+ cluster_multiple_rels(rvs, params.options, true);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -245,7 +270,7 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
* cluster_context.
*/
rvs = get_tables_to_cluster(cluster_context);
- cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+ cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -282,14 +307,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* Check for user-requested abort. */
CHECK_FOR_INTERRUPTS();
- pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
- if (OidIsValid(indexOid))
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_CLUSTER);
- else
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
/*
* We grab exclusive access to the target rel and index for the duration
* of the transaction. (This is redundant for the single-transaction
@@ -300,10 +317,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* If the table has gone away, we can skip processing it */
if (!OldHeap)
- {
- pgstat_progress_end_command();
return;
- }
/*
* Since we may open a new transaction for each relation, we have to check
@@ -319,7 +333,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!pg_class_ownercheck(tableOid, GetUserId()))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -334,7 +347,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (RELATION_IS_OTHER_TEMP(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -346,7 +358,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -357,7 +368,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!get_index_isclustered(indexOid))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
}
@@ -416,7 +426,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!RelationIsPopulated(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -424,7 +433,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -440,8 +448,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
rebuild_relation(OldHeap, indexOid, verbose);
/* NB: rebuild_relation does table_close() on OldHeap */
-
- pgstat_progress_end_command();
}
/*
@@ -1366,7 +1372,7 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap,
ReindexParams reindex_params = {0};
int i;
- /* Report that we are now swapping relation files */
+ /* Report that we are now swapping relation files XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_SWAP_REL_FILES);
@@ -1417,13 +1423,13 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap,
else if (newrelpersistence == RELPERSISTENCE_PERMANENT)
reindex_flags |= REINDEX_REL_FORCE_INDEXES_PERMANENT;
- /* Report that we are now reindexing relations */
+ /* Report that we are now reindexing relations XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_REBUILD_INDEX);
reindex_relation(OIDOldHeap, reindex_flags, &reindex_params);
- /* Report that we are now doing clean up */
+ /* Report that we are now doing clean up XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_FINAL_CLEANUP);
@@ -1633,9 +1639,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
/* Cluster each relation in a separate transaction */
static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
{
ListCell *lc;
+ off_t childnum = 0;
/* Commit to get out of starting transaction */
PopActiveSnapshot();
@@ -1658,6 +1665,11 @@ cluster_multiple_rels(List *rvs, int options)
cluster_rel(rvtc->tableOid, rvtc->indexOid,
&cluster_params);
+ if (ispartitioned)
+ pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+ childnum++);
+
+
PopActiveSnapshot();
CommitTransactionCommand();
}
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
#define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS 5
#define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED 6
#define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT 7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL 17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE 18
/* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
#define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP 1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b632d9f2ea..d1fea532bc 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1941,7 +1941,9 @@ pg_stat_progress_cluster| SELECT s.pid,
s.param5 AS heap_tuples_written,
s.param6 AS heap_blks_total,
s.param7 AS heap_blks_scanned,
- s.param8 AS index_rebuild_count
+ s.param8 AS index_rebuild_count,
+ s.param18 AS partitions_total,
+ s.param19 AS partitions_done
FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
LEFT JOIN pg_database d ON ((s.datid = d.oid)));
pg_stat_progress_copy| SELECT s.pid,
--
2.17.0
--O5XBE6gyVG5Rl6Rj
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v8-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"
^ permalink raw reply [nested|flat] 73+ messages in thread
* [PATCH v8 3/8] f! progress reporting..
@ 2021-02-09 02:45 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 73+ messages in thread
From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)
This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.
This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
src/backend/catalog/system_views.sql | 4 +-
src/backend/commands/cluster.c | 64 +++++++++++++++++-----------
src/include/commands/progress.h | 3 ++
src/test/regress/expected/rules.out | 4 +-
4 files changed, 47 insertions(+), 28 deletions(-)
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fa58afd9d7..9a3cb5f935 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1069,7 +1069,9 @@ CREATE VIEW pg_stat_progress_cluster AS
S.param5 AS heap_tuples_written,
S.param6 AS heap_blks_total,
S.param7 AS heap_blks_scanned,
- S.param8 AS index_rebuild_count
+ S.param8 AS index_rebuild_count,
+ S.param18 AS partitions_total,
+ S.param19 AS partitions_done
FROM pg_stat_get_progress_info('CLUSTER') AS S
LEFT JOIN pg_database D ON S.datid = D.oid;
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 9b6673867c..cb4fc350c6 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
static List *get_tables_to_cluster(MemoryContext cluster_context);
static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
/*---------------------------------------------------------------------------
@@ -188,16 +188,28 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
/* close relation, keep lock till commit */
table_close(rel, ShareUpdateExclusiveLock);
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
{
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ OidIsValid(indexOid) ? PROGRESS_CLUSTER_COMMAND_CLUSTER :
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
+
/* Do the job. */
cluster_rel(tableOid, indexOid, ¶ms);
+ pgstat_progress_end_command();
}
else
{
List *rvs;
MemoryContext cluster_context;
+ int64 progress_val[2];
+ const int progress_index[] = {
+ PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+ };
+
/* Refuse to hold strong locks in a user transaction */
PreventInTransactionBlock(isTopLevel, "CLUSTER");
@@ -206,7 +218,20 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
ALLOCSET_DEFAULT_SIZES);
rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
- cluster_multiple_rels(rvs, params.options);
+
+ /* Report vacuum full or cluster and number of partitions */
+ progress_val[0] = OidIsValid(indexOid) ?
+ PROGRESS_CLUSTER_COMMAND_CLUSTER :
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL;
+ progress_val[1] = list_length(rvs);
+// This is currently showing the total number of *tables*, including the
+// parent, and intermediate partitions, so the column should be renamed, or
+// we should count the number of leaf partitions
+
+ pgstat_progress_update_multi_param(2, progress_index, progress_val);
+
+ cluster_multiple_rels(rvs, params.options, true);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -245,7 +270,7 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
* cluster_context.
*/
rvs = get_tables_to_cluster(cluster_context);
- cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+ cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -282,14 +307,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* Check for user-requested abort. */
CHECK_FOR_INTERRUPTS();
- pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
- if (OidIsValid(indexOid))
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_CLUSTER);
- else
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
/*
* We grab exclusive access to the target rel and index for the duration
* of the transaction. (This is redundant for the single-transaction
@@ -300,10 +317,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* If the table has gone away, we can skip processing it */
if (!OldHeap)
- {
- pgstat_progress_end_command();
return;
- }
/*
* Since we may open a new transaction for each relation, we have to check
@@ -319,7 +333,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!pg_class_ownercheck(tableOid, GetUserId()))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -334,7 +347,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (RELATION_IS_OTHER_TEMP(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -346,7 +358,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -357,7 +368,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!get_index_isclustered(indexOid))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
}
@@ -416,7 +426,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!RelationIsPopulated(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -424,7 +433,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -440,8 +448,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
rebuild_relation(OldHeap, indexOid, verbose);
/* NB: rebuild_relation does table_close() on OldHeap */
-
- pgstat_progress_end_command();
}
/*
@@ -1366,7 +1372,7 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap,
ReindexParams reindex_params = {0};
int i;
- /* Report that we are now swapping relation files */
+ /* Report that we are now swapping relation files XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_SWAP_REL_FILES);
@@ -1417,13 +1423,13 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap,
else if (newrelpersistence == RELPERSISTENCE_PERMANENT)
reindex_flags |= REINDEX_REL_FORCE_INDEXES_PERMANENT;
- /* Report that we are now reindexing relations */
+ /* Report that we are now reindexing relations XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_REBUILD_INDEX);
reindex_relation(OIDOldHeap, reindex_flags, &reindex_params);
- /* Report that we are now doing clean up */
+ /* Report that we are now doing clean up XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_FINAL_CLEANUP);
@@ -1633,9 +1639,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
/* Cluster each relation in a separate transaction */
static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
{
ListCell *lc;
+ off_t childnum = 0;
/* Commit to get out of starting transaction */
PopActiveSnapshot();
@@ -1658,6 +1665,11 @@ cluster_multiple_rels(List *rvs, int options)
cluster_rel(rvtc->tableOid, rvtc->indexOid,
&cluster_params);
+ if (ispartitioned)
+ pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+ childnum++);
+
+
PopActiveSnapshot();
CommitTransactionCommand();
}
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
#define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS 5
#define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED 6
#define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT 7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL 17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE 18
/* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
#define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP 1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b632d9f2ea..d1fea532bc 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1941,7 +1941,9 @@ pg_stat_progress_cluster| SELECT s.pid,
s.param5 AS heap_tuples_written,
s.param6 AS heap_blks_total,
s.param7 AS heap_blks_scanned,
- s.param8 AS index_rebuild_count
+ s.param8 AS index_rebuild_count,
+ s.param18 AS partitions_total,
+ s.param19 AS partitions_done
FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
LEFT JOIN pg_database d ON ((s.datid = d.oid)));
pg_stat_progress_copy| SELECT s.pid,
--
2.17.0
--O5XBE6gyVG5Rl6Rj
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v8-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"
^ permalink raw reply [nested|flat] 73+ messages in thread
* [PATCH v9 3/8] f! progress reporting..
@ 2021-02-09 02:45 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 73+ messages in thread
From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)
This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.
This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
doc/src/sgml/monitoring.sgml | 21 +++++++++++
src/backend/catalog/system_views.sql | 4 +-
src/backend/commands/cluster.c | 55 ++++++++++++++++------------
src/backend/commands/vacuum.c | 5 +++
src/include/commands/progress.h | 3 ++
src/test/regress/expected/rules.out | 4 +-
6 files changed, 67 insertions(+), 25 deletions(-)
diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml
index 3513e127b7..dca47f6a48 100644
--- a/doc/src/sgml/monitoring.sgml
+++ b/doc/src/sgml/monitoring.sgml
@@ -6231,6 +6231,27 @@ SELECT pg_stat_get_backend_pid(s.backendid) AS pid,
is <literal>rebuilding index</literal>.
</para></entry>
</row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>total_partitions</structfield> <type>bigint</type>
+ </para>
+ When clustering a partitioned table, this column is set to the total
+ number of partitions to be clustered.
+ <para>
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>done_partitions</structfield> <type>bigint</type>
+ </para>
+ <para>
+ When clustering a partitioned table, this column is set to the number
+ of partitions which have been clustered.
+ </para></entry>
+ </row>
+
</tbody>
</tgroup>
</table>
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fc94a73a54..21c27f0437 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1070,7 +1070,9 @@ CREATE VIEW pg_stat_progress_cluster AS
S.param5 AS heap_tuples_written,
S.param6 AS heap_blks_total,
S.param7 AS heap_blks_scanned,
- S.param8 AS index_rebuild_count
+ S.param8 AS index_rebuild_count,
+ S.param18 AS partitions_total,
+ S.param19 AS partitions_done
FROM pg_stat_get_progress_info('CLUSTER') AS S
LEFT JOIN pg_database D ON S.datid = D.oid;
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 0c08ac56dc..7d8d1d8a69 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
static List *get_tables_to_cluster(MemoryContext cluster_context);
static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
/*---------------------------------------------------------------------------
@@ -188,16 +188,26 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
/* close relation, keep lock till commit */
table_close(rel, ShareUpdateExclusiveLock);
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
{
/* Do the job. */
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_CLUSTER);
cluster_rel(tableOid, indexOid, ¶ms);
+ pgstat_progress_end_command();
}
else
{
List *rvs;
MemoryContext cluster_context;
+ int64 progress_val[2];
+ const int progress_index[2] = {
+ PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+ };
+
/* Refuse to hold strong locks in a user transaction */
PreventInTransactionBlock(isTopLevel, "CLUSTER");
@@ -206,7 +216,15 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
ALLOCSET_DEFAULT_SIZES);
rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
- cluster_multiple_rels(rvs, params.options);
+
+ /* Report command and number of partitions */
+ progress_val[0] = PROGRESS_CLUSTER_COMMAND_CLUSTER;
+ /* XXX: rvs includes the parent, which is not a "partition" */
+ progress_val[1] = list_length(rvs);
+
+ pgstat_progress_update_multi_param(2, progress_index, progress_val);
+ cluster_multiple_rels(rvs, params.options, true);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -245,7 +263,11 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
* cluster_context.
*/
rvs = get_tables_to_cluster(cluster_context);
- cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_CLUSTER);
+ cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -282,14 +304,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* Check for user-requested abort. */
CHECK_FOR_INTERRUPTS();
- pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
- if (OidIsValid(indexOid))
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_CLUSTER);
- else
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
/*
* We grab exclusive access to the target rel and index for the duration
* of the transaction. (This is redundant for the single-transaction
@@ -300,10 +314,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* If the table has gone away, we can skip processing it */
if (!OldHeap)
- {
- pgstat_progress_end_command();
return;
- }
/*
* Since we may open a new transaction for each relation, we have to check
@@ -319,7 +330,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!pg_class_ownercheck(tableOid, GetUserId()))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -334,7 +344,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (RELATION_IS_OTHER_TEMP(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -346,7 +355,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -357,7 +365,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!get_index_isclustered(indexOid))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
}
@@ -416,7 +423,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!RelationIsPopulated(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -424,7 +430,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -440,8 +445,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
rebuild_relation(OldHeap, indexOid, verbose);
/* NB: rebuild_relation does table_close() on OldHeap */
-
- pgstat_progress_end_command();
}
/*
@@ -1632,9 +1635,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
/* Cluster each relation in a separate transaction */
static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
{
ListCell *lc;
+ off_t childnum = 0;
/* Commit to get out of starting transaction */
PopActiveSnapshot();
@@ -1657,6 +1661,11 @@ cluster_multiple_rels(List *rvs, int options)
cluster_rel(rvtc->tableOid, rvtc->indexOid,
&cluster_params);
+ if (ispartitioned)
+ pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+ ++childnum);
+
+
PopActiveSnapshot();
CommitTransactionCommand();
}
diff --git a/src/backend/commands/vacuum.c b/src/backend/commands/vacuum.c
index c064352e23..05d52858b1 100644
--- a/src/backend/commands/vacuum.c
+++ b/src/backend/commands/vacuum.c
@@ -37,6 +37,7 @@
#include "catalog/pg_namespace.h"
#include "commands/cluster.h"
#include "commands/defrem.h"
+#include "commands/progress.h"
#include "commands/vacuum.h"
#include "miscadmin.h"
#include "nodes/makefuncs.h"
@@ -1937,7 +1938,11 @@ vacuum_rel(Oid relid, RangeVar *relation, VacuumParams *params)
cluster_params.options |= CLUOPT_VERBOSE;
/* VACUUM FULL is now a variant of CLUSTER; see cluster.c */
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, relid);
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
cluster_rel(relid, InvalidOid, &cluster_params);
+ pgstat_progress_end_command();
}
else
table_relation_vacuum(onerel, params, vac_strategy);
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
#define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS 5
#define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED 6
#define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT 7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL 17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE 18
/* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
#define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP 1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b1c9b7bdfe..c888415235 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1943,7 +1943,9 @@ pg_stat_progress_cluster| SELECT s.pid,
s.param5 AS heap_tuples_written,
s.param6 AS heap_blks_total,
s.param7 AS heap_blks_scanned,
- s.param8 AS index_rebuild_count
+ s.param8 AS index_rebuild_count,
+ s.param18 AS partitions_total,
+ s.param19 AS partitions_done
FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
LEFT JOIN pg_database d ON ((s.datid = d.oid)));
pg_stat_progress_copy| SELECT s.pid,
--
2.17.0
--fmvA4kSBHQVZhkR6
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v9-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"
^ permalink raw reply [nested|flat] 73+ messages in thread
* [PATCH v8 3/8] f! progress reporting..
@ 2021-02-09 02:45 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 73+ messages in thread
From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)
This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.
This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
src/backend/catalog/system_views.sql | 4 +-
src/backend/commands/cluster.c | 64 +++++++++++++++++-----------
src/include/commands/progress.h | 3 ++
src/test/regress/expected/rules.out | 4 +-
4 files changed, 47 insertions(+), 28 deletions(-)
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fa58afd9d7..9a3cb5f935 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1069,7 +1069,9 @@ CREATE VIEW pg_stat_progress_cluster AS
S.param5 AS heap_tuples_written,
S.param6 AS heap_blks_total,
S.param7 AS heap_blks_scanned,
- S.param8 AS index_rebuild_count
+ S.param8 AS index_rebuild_count,
+ S.param18 AS partitions_total,
+ S.param19 AS partitions_done
FROM pg_stat_get_progress_info('CLUSTER') AS S
LEFT JOIN pg_database D ON S.datid = D.oid;
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 9b6673867c..cb4fc350c6 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
static List *get_tables_to_cluster(MemoryContext cluster_context);
static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
/*---------------------------------------------------------------------------
@@ -188,16 +188,28 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
/* close relation, keep lock till commit */
table_close(rel, ShareUpdateExclusiveLock);
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
{
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ OidIsValid(indexOid) ? PROGRESS_CLUSTER_COMMAND_CLUSTER :
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
+
/* Do the job. */
cluster_rel(tableOid, indexOid, ¶ms);
+ pgstat_progress_end_command();
}
else
{
List *rvs;
MemoryContext cluster_context;
+ int64 progress_val[2];
+ const int progress_index[] = {
+ PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+ };
+
/* Refuse to hold strong locks in a user transaction */
PreventInTransactionBlock(isTopLevel, "CLUSTER");
@@ -206,7 +218,20 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
ALLOCSET_DEFAULT_SIZES);
rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
- cluster_multiple_rels(rvs, params.options);
+
+ /* Report vacuum full or cluster and number of partitions */
+ progress_val[0] = OidIsValid(indexOid) ?
+ PROGRESS_CLUSTER_COMMAND_CLUSTER :
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL;
+ progress_val[1] = list_length(rvs);
+// This is currently showing the total number of *tables*, including the
+// parent, and intermediate partitions, so the column should be renamed, or
+// we should count the number of leaf partitions
+
+ pgstat_progress_update_multi_param(2, progress_index, progress_val);
+
+ cluster_multiple_rels(rvs, params.options, true);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -245,7 +270,7 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
* cluster_context.
*/
rvs = get_tables_to_cluster(cluster_context);
- cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+ cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -282,14 +307,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* Check for user-requested abort. */
CHECK_FOR_INTERRUPTS();
- pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
- if (OidIsValid(indexOid))
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_CLUSTER);
- else
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
/*
* We grab exclusive access to the target rel and index for the duration
* of the transaction. (This is redundant for the single-transaction
@@ -300,10 +317,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* If the table has gone away, we can skip processing it */
if (!OldHeap)
- {
- pgstat_progress_end_command();
return;
- }
/*
* Since we may open a new transaction for each relation, we have to check
@@ -319,7 +333,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!pg_class_ownercheck(tableOid, GetUserId()))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -334,7 +347,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (RELATION_IS_OTHER_TEMP(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -346,7 +358,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -357,7 +368,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!get_index_isclustered(indexOid))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
}
@@ -416,7 +426,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!RelationIsPopulated(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -424,7 +433,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -440,8 +448,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
rebuild_relation(OldHeap, indexOid, verbose);
/* NB: rebuild_relation does table_close() on OldHeap */
-
- pgstat_progress_end_command();
}
/*
@@ -1366,7 +1372,7 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap,
ReindexParams reindex_params = {0};
int i;
- /* Report that we are now swapping relation files */
+ /* Report that we are now swapping relation files XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_SWAP_REL_FILES);
@@ -1417,13 +1423,13 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap,
else if (newrelpersistence == RELPERSISTENCE_PERMANENT)
reindex_flags |= REINDEX_REL_FORCE_INDEXES_PERMANENT;
- /* Report that we are now reindexing relations */
+ /* Report that we are now reindexing relations XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_REBUILD_INDEX);
reindex_relation(OIDOldHeap, reindex_flags, &reindex_params);
- /* Report that we are now doing clean up */
+ /* Report that we are now doing clean up XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_FINAL_CLEANUP);
@@ -1633,9 +1639,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
/* Cluster each relation in a separate transaction */
static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
{
ListCell *lc;
+ off_t childnum = 0;
/* Commit to get out of starting transaction */
PopActiveSnapshot();
@@ -1658,6 +1665,11 @@ cluster_multiple_rels(List *rvs, int options)
cluster_rel(rvtc->tableOid, rvtc->indexOid,
&cluster_params);
+ if (ispartitioned)
+ pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+ childnum++);
+
+
PopActiveSnapshot();
CommitTransactionCommand();
}
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
#define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS 5
#define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED 6
#define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT 7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL 17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE 18
/* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
#define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP 1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b632d9f2ea..d1fea532bc 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1941,7 +1941,9 @@ pg_stat_progress_cluster| SELECT s.pid,
s.param5 AS heap_tuples_written,
s.param6 AS heap_blks_total,
s.param7 AS heap_blks_scanned,
- s.param8 AS index_rebuild_count
+ s.param8 AS index_rebuild_count,
+ s.param18 AS partitions_total,
+ s.param19 AS partitions_done
FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
LEFT JOIN pg_database d ON ((s.datid = d.oid)));
pg_stat_progress_copy| SELECT s.pid,
--
2.17.0
--O5XBE6gyVG5Rl6Rj
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v8-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"
^ permalink raw reply [nested|flat] 73+ messages in thread
* [PATCH v9 3/8] f! progress reporting..
@ 2021-02-09 02:45 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 73+ messages in thread
From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)
This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.
This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
doc/src/sgml/monitoring.sgml | 21 +++++++++++
src/backend/catalog/system_views.sql | 4 +-
src/backend/commands/cluster.c | 55 ++++++++++++++++------------
src/backend/commands/vacuum.c | 5 +++
src/include/commands/progress.h | 3 ++
src/test/regress/expected/rules.out | 4 +-
6 files changed, 67 insertions(+), 25 deletions(-)
diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml
index 3513e127b7..dca47f6a48 100644
--- a/doc/src/sgml/monitoring.sgml
+++ b/doc/src/sgml/monitoring.sgml
@@ -6231,6 +6231,27 @@ SELECT pg_stat_get_backend_pid(s.backendid) AS pid,
is <literal>rebuilding index</literal>.
</para></entry>
</row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>total_partitions</structfield> <type>bigint</type>
+ </para>
+ When clustering a partitioned table, this column is set to the total
+ number of partitions to be clustered.
+ <para>
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>done_partitions</structfield> <type>bigint</type>
+ </para>
+ <para>
+ When clustering a partitioned table, this column is set to the number
+ of partitions which have been clustered.
+ </para></entry>
+ </row>
+
</tbody>
</tgroup>
</table>
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fc94a73a54..21c27f0437 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1070,7 +1070,9 @@ CREATE VIEW pg_stat_progress_cluster AS
S.param5 AS heap_tuples_written,
S.param6 AS heap_blks_total,
S.param7 AS heap_blks_scanned,
- S.param8 AS index_rebuild_count
+ S.param8 AS index_rebuild_count,
+ S.param18 AS partitions_total,
+ S.param19 AS partitions_done
FROM pg_stat_get_progress_info('CLUSTER') AS S
LEFT JOIN pg_database D ON S.datid = D.oid;
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 0c08ac56dc..7d8d1d8a69 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
static List *get_tables_to_cluster(MemoryContext cluster_context);
static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
/*---------------------------------------------------------------------------
@@ -188,16 +188,26 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
/* close relation, keep lock till commit */
table_close(rel, ShareUpdateExclusiveLock);
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
{
/* Do the job. */
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_CLUSTER);
cluster_rel(tableOid, indexOid, ¶ms);
+ pgstat_progress_end_command();
}
else
{
List *rvs;
MemoryContext cluster_context;
+ int64 progress_val[2];
+ const int progress_index[2] = {
+ PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+ };
+
/* Refuse to hold strong locks in a user transaction */
PreventInTransactionBlock(isTopLevel, "CLUSTER");
@@ -206,7 +216,15 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
ALLOCSET_DEFAULT_SIZES);
rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
- cluster_multiple_rels(rvs, params.options);
+
+ /* Report command and number of partitions */
+ progress_val[0] = PROGRESS_CLUSTER_COMMAND_CLUSTER;
+ /* XXX: rvs includes the parent, which is not a "partition" */
+ progress_val[1] = list_length(rvs);
+
+ pgstat_progress_update_multi_param(2, progress_index, progress_val);
+ cluster_multiple_rels(rvs, params.options, true);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -245,7 +263,11 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
* cluster_context.
*/
rvs = get_tables_to_cluster(cluster_context);
- cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_CLUSTER);
+ cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -282,14 +304,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* Check for user-requested abort. */
CHECK_FOR_INTERRUPTS();
- pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
- if (OidIsValid(indexOid))
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_CLUSTER);
- else
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
/*
* We grab exclusive access to the target rel and index for the duration
* of the transaction. (This is redundant for the single-transaction
@@ -300,10 +314,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* If the table has gone away, we can skip processing it */
if (!OldHeap)
- {
- pgstat_progress_end_command();
return;
- }
/*
* Since we may open a new transaction for each relation, we have to check
@@ -319,7 +330,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!pg_class_ownercheck(tableOid, GetUserId()))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -334,7 +344,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (RELATION_IS_OTHER_TEMP(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -346,7 +355,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -357,7 +365,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!get_index_isclustered(indexOid))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
}
@@ -416,7 +423,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!RelationIsPopulated(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -424,7 +430,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -440,8 +445,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
rebuild_relation(OldHeap, indexOid, verbose);
/* NB: rebuild_relation does table_close() on OldHeap */
-
- pgstat_progress_end_command();
}
/*
@@ -1632,9 +1635,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
/* Cluster each relation in a separate transaction */
static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
{
ListCell *lc;
+ off_t childnum = 0;
/* Commit to get out of starting transaction */
PopActiveSnapshot();
@@ -1657,6 +1661,11 @@ cluster_multiple_rels(List *rvs, int options)
cluster_rel(rvtc->tableOid, rvtc->indexOid,
&cluster_params);
+ if (ispartitioned)
+ pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+ ++childnum);
+
+
PopActiveSnapshot();
CommitTransactionCommand();
}
diff --git a/src/backend/commands/vacuum.c b/src/backend/commands/vacuum.c
index c064352e23..05d52858b1 100644
--- a/src/backend/commands/vacuum.c
+++ b/src/backend/commands/vacuum.c
@@ -37,6 +37,7 @@
#include "catalog/pg_namespace.h"
#include "commands/cluster.h"
#include "commands/defrem.h"
+#include "commands/progress.h"
#include "commands/vacuum.h"
#include "miscadmin.h"
#include "nodes/makefuncs.h"
@@ -1937,7 +1938,11 @@ vacuum_rel(Oid relid, RangeVar *relation, VacuumParams *params)
cluster_params.options |= CLUOPT_VERBOSE;
/* VACUUM FULL is now a variant of CLUSTER; see cluster.c */
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, relid);
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
cluster_rel(relid, InvalidOid, &cluster_params);
+ pgstat_progress_end_command();
}
else
table_relation_vacuum(onerel, params, vac_strategy);
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
#define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS 5
#define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED 6
#define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT 7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL 17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE 18
/* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
#define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP 1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b1c9b7bdfe..c888415235 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1943,7 +1943,9 @@ pg_stat_progress_cluster| SELECT s.pid,
s.param5 AS heap_tuples_written,
s.param6 AS heap_blks_total,
s.param7 AS heap_blks_scanned,
- s.param8 AS index_rebuild_count
+ s.param8 AS index_rebuild_count,
+ s.param18 AS partitions_total,
+ s.param19 AS partitions_done
FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
LEFT JOIN pg_database d ON ((s.datid = d.oid)));
pg_stat_progress_copy| SELECT s.pid,
--
2.17.0
--fmvA4kSBHQVZhkR6
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v9-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"
^ permalink raw reply [nested|flat] 73+ messages in thread
* [PATCH v9 3/8] f! progress reporting..
@ 2021-02-09 02:45 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 73+ messages in thread
From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)
This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.
This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
doc/src/sgml/monitoring.sgml | 21 +++++++++++
src/backend/catalog/system_views.sql | 4 +-
src/backend/commands/cluster.c | 55 ++++++++++++++++------------
src/backend/commands/vacuum.c | 5 +++
src/include/commands/progress.h | 3 ++
src/test/regress/expected/rules.out | 4 +-
6 files changed, 67 insertions(+), 25 deletions(-)
diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml
index 3513e127b7..dca47f6a48 100644
--- a/doc/src/sgml/monitoring.sgml
+++ b/doc/src/sgml/monitoring.sgml
@@ -6231,6 +6231,27 @@ SELECT pg_stat_get_backend_pid(s.backendid) AS pid,
is <literal>rebuilding index</literal>.
</para></entry>
</row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>total_partitions</structfield> <type>bigint</type>
+ </para>
+ When clustering a partitioned table, this column is set to the total
+ number of partitions to be clustered.
+ <para>
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>done_partitions</structfield> <type>bigint</type>
+ </para>
+ <para>
+ When clustering a partitioned table, this column is set to the number
+ of partitions which have been clustered.
+ </para></entry>
+ </row>
+
</tbody>
</tgroup>
</table>
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fc94a73a54..21c27f0437 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1070,7 +1070,9 @@ CREATE VIEW pg_stat_progress_cluster AS
S.param5 AS heap_tuples_written,
S.param6 AS heap_blks_total,
S.param7 AS heap_blks_scanned,
- S.param8 AS index_rebuild_count
+ S.param8 AS index_rebuild_count,
+ S.param18 AS partitions_total,
+ S.param19 AS partitions_done
FROM pg_stat_get_progress_info('CLUSTER') AS S
LEFT JOIN pg_database D ON S.datid = D.oid;
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 0c08ac56dc..7d8d1d8a69 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
static List *get_tables_to_cluster(MemoryContext cluster_context);
static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
/*---------------------------------------------------------------------------
@@ -188,16 +188,26 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
/* close relation, keep lock till commit */
table_close(rel, ShareUpdateExclusiveLock);
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
{
/* Do the job. */
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_CLUSTER);
cluster_rel(tableOid, indexOid, ¶ms);
+ pgstat_progress_end_command();
}
else
{
List *rvs;
MemoryContext cluster_context;
+ int64 progress_val[2];
+ const int progress_index[2] = {
+ PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+ };
+
/* Refuse to hold strong locks in a user transaction */
PreventInTransactionBlock(isTopLevel, "CLUSTER");
@@ -206,7 +216,15 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
ALLOCSET_DEFAULT_SIZES);
rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
- cluster_multiple_rels(rvs, params.options);
+
+ /* Report command and number of partitions */
+ progress_val[0] = PROGRESS_CLUSTER_COMMAND_CLUSTER;
+ /* XXX: rvs includes the parent, which is not a "partition" */
+ progress_val[1] = list_length(rvs);
+
+ pgstat_progress_update_multi_param(2, progress_index, progress_val);
+ cluster_multiple_rels(rvs, params.options, true);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -245,7 +263,11 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
* cluster_context.
*/
rvs = get_tables_to_cluster(cluster_context);
- cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_CLUSTER);
+ cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -282,14 +304,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* Check for user-requested abort. */
CHECK_FOR_INTERRUPTS();
- pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
- if (OidIsValid(indexOid))
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_CLUSTER);
- else
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
/*
* We grab exclusive access to the target rel and index for the duration
* of the transaction. (This is redundant for the single-transaction
@@ -300,10 +314,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* If the table has gone away, we can skip processing it */
if (!OldHeap)
- {
- pgstat_progress_end_command();
return;
- }
/*
* Since we may open a new transaction for each relation, we have to check
@@ -319,7 +330,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!pg_class_ownercheck(tableOid, GetUserId()))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -334,7 +344,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (RELATION_IS_OTHER_TEMP(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -346,7 +355,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -357,7 +365,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!get_index_isclustered(indexOid))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
}
@@ -416,7 +423,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!RelationIsPopulated(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -424,7 +430,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -440,8 +445,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
rebuild_relation(OldHeap, indexOid, verbose);
/* NB: rebuild_relation does table_close() on OldHeap */
-
- pgstat_progress_end_command();
}
/*
@@ -1632,9 +1635,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
/* Cluster each relation in a separate transaction */
static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
{
ListCell *lc;
+ off_t childnum = 0;
/* Commit to get out of starting transaction */
PopActiveSnapshot();
@@ -1657,6 +1661,11 @@ cluster_multiple_rels(List *rvs, int options)
cluster_rel(rvtc->tableOid, rvtc->indexOid,
&cluster_params);
+ if (ispartitioned)
+ pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+ ++childnum);
+
+
PopActiveSnapshot();
CommitTransactionCommand();
}
diff --git a/src/backend/commands/vacuum.c b/src/backend/commands/vacuum.c
index c064352e23..05d52858b1 100644
--- a/src/backend/commands/vacuum.c
+++ b/src/backend/commands/vacuum.c
@@ -37,6 +37,7 @@
#include "catalog/pg_namespace.h"
#include "commands/cluster.h"
#include "commands/defrem.h"
+#include "commands/progress.h"
#include "commands/vacuum.h"
#include "miscadmin.h"
#include "nodes/makefuncs.h"
@@ -1937,7 +1938,11 @@ vacuum_rel(Oid relid, RangeVar *relation, VacuumParams *params)
cluster_params.options |= CLUOPT_VERBOSE;
/* VACUUM FULL is now a variant of CLUSTER; see cluster.c */
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, relid);
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
cluster_rel(relid, InvalidOid, &cluster_params);
+ pgstat_progress_end_command();
}
else
table_relation_vacuum(onerel, params, vac_strategy);
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
#define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS 5
#define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED 6
#define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT 7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL 17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE 18
/* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
#define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP 1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b1c9b7bdfe..c888415235 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1943,7 +1943,9 @@ pg_stat_progress_cluster| SELECT s.pid,
s.param5 AS heap_tuples_written,
s.param6 AS heap_blks_total,
s.param7 AS heap_blks_scanned,
- s.param8 AS index_rebuild_count
+ s.param8 AS index_rebuild_count,
+ s.param18 AS partitions_total,
+ s.param19 AS partitions_done
FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
LEFT JOIN pg_database d ON ((s.datid = d.oid)));
pg_stat_progress_copy| SELECT s.pid,
--
2.17.0
--fmvA4kSBHQVZhkR6
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v9-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"
^ permalink raw reply [nested|flat] 73+ messages in thread
* [PATCH v8 3/8] f! progress reporting..
@ 2021-02-09 02:45 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 73+ messages in thread
From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)
This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.
This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
src/backend/catalog/system_views.sql | 4 +-
src/backend/commands/cluster.c | 64 +++++++++++++++++-----------
src/include/commands/progress.h | 3 ++
src/test/regress/expected/rules.out | 4 +-
4 files changed, 47 insertions(+), 28 deletions(-)
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fa58afd9d7..9a3cb5f935 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1069,7 +1069,9 @@ CREATE VIEW pg_stat_progress_cluster AS
S.param5 AS heap_tuples_written,
S.param6 AS heap_blks_total,
S.param7 AS heap_blks_scanned,
- S.param8 AS index_rebuild_count
+ S.param8 AS index_rebuild_count,
+ S.param18 AS partitions_total,
+ S.param19 AS partitions_done
FROM pg_stat_get_progress_info('CLUSTER') AS S
LEFT JOIN pg_database D ON S.datid = D.oid;
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 9b6673867c..cb4fc350c6 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
static List *get_tables_to_cluster(MemoryContext cluster_context);
static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
/*---------------------------------------------------------------------------
@@ -188,16 +188,28 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
/* close relation, keep lock till commit */
table_close(rel, ShareUpdateExclusiveLock);
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
{
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ OidIsValid(indexOid) ? PROGRESS_CLUSTER_COMMAND_CLUSTER :
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
+
/* Do the job. */
cluster_rel(tableOid, indexOid, ¶ms);
+ pgstat_progress_end_command();
}
else
{
List *rvs;
MemoryContext cluster_context;
+ int64 progress_val[2];
+ const int progress_index[] = {
+ PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+ };
+
/* Refuse to hold strong locks in a user transaction */
PreventInTransactionBlock(isTopLevel, "CLUSTER");
@@ -206,7 +218,20 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
ALLOCSET_DEFAULT_SIZES);
rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
- cluster_multiple_rels(rvs, params.options);
+
+ /* Report vacuum full or cluster and number of partitions */
+ progress_val[0] = OidIsValid(indexOid) ?
+ PROGRESS_CLUSTER_COMMAND_CLUSTER :
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL;
+ progress_val[1] = list_length(rvs);
+// This is currently showing the total number of *tables*, including the
+// parent, and intermediate partitions, so the column should be renamed, or
+// we should count the number of leaf partitions
+
+ pgstat_progress_update_multi_param(2, progress_index, progress_val);
+
+ cluster_multiple_rels(rvs, params.options, true);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -245,7 +270,7 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
* cluster_context.
*/
rvs = get_tables_to_cluster(cluster_context);
- cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+ cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -282,14 +307,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* Check for user-requested abort. */
CHECK_FOR_INTERRUPTS();
- pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
- if (OidIsValid(indexOid))
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_CLUSTER);
- else
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
/*
* We grab exclusive access to the target rel and index for the duration
* of the transaction. (This is redundant for the single-transaction
@@ -300,10 +317,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* If the table has gone away, we can skip processing it */
if (!OldHeap)
- {
- pgstat_progress_end_command();
return;
- }
/*
* Since we may open a new transaction for each relation, we have to check
@@ -319,7 +333,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!pg_class_ownercheck(tableOid, GetUserId()))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -334,7 +347,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (RELATION_IS_OTHER_TEMP(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -346,7 +358,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -357,7 +368,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!get_index_isclustered(indexOid))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
}
@@ -416,7 +426,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!RelationIsPopulated(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -424,7 +433,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -440,8 +448,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
rebuild_relation(OldHeap, indexOid, verbose);
/* NB: rebuild_relation does table_close() on OldHeap */
-
- pgstat_progress_end_command();
}
/*
@@ -1366,7 +1372,7 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap,
ReindexParams reindex_params = {0};
int i;
- /* Report that we are now swapping relation files */
+ /* Report that we are now swapping relation files XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_SWAP_REL_FILES);
@@ -1417,13 +1423,13 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap,
else if (newrelpersistence == RELPERSISTENCE_PERMANENT)
reindex_flags |= REINDEX_REL_FORCE_INDEXES_PERMANENT;
- /* Report that we are now reindexing relations */
+ /* Report that we are now reindexing relations XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_REBUILD_INDEX);
reindex_relation(OIDOldHeap, reindex_flags, &reindex_params);
- /* Report that we are now doing clean up */
+ /* Report that we are now doing clean up XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_FINAL_CLEANUP);
@@ -1633,9 +1639,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
/* Cluster each relation in a separate transaction */
static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
{
ListCell *lc;
+ off_t childnum = 0;
/* Commit to get out of starting transaction */
PopActiveSnapshot();
@@ -1658,6 +1665,11 @@ cluster_multiple_rels(List *rvs, int options)
cluster_rel(rvtc->tableOid, rvtc->indexOid,
&cluster_params);
+ if (ispartitioned)
+ pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+ childnum++);
+
+
PopActiveSnapshot();
CommitTransactionCommand();
}
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
#define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS 5
#define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED 6
#define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT 7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL 17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE 18
/* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
#define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP 1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b632d9f2ea..d1fea532bc 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1941,7 +1941,9 @@ pg_stat_progress_cluster| SELECT s.pid,
s.param5 AS heap_tuples_written,
s.param6 AS heap_blks_total,
s.param7 AS heap_blks_scanned,
- s.param8 AS index_rebuild_count
+ s.param8 AS index_rebuild_count,
+ s.param18 AS partitions_total,
+ s.param19 AS partitions_done
FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
LEFT JOIN pg_database d ON ((s.datid = d.oid)));
pg_stat_progress_copy| SELECT s.pid,
--
2.17.0
--O5XBE6gyVG5Rl6Rj
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v8-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"
^ permalink raw reply [nested|flat] 73+ messages in thread
* [PATCH v9 3/8] f! progress reporting..
@ 2021-02-09 02:45 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 73+ messages in thread
From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)
This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.
This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
doc/src/sgml/monitoring.sgml | 21 +++++++++++
src/backend/catalog/system_views.sql | 4 +-
src/backend/commands/cluster.c | 55 ++++++++++++++++------------
src/backend/commands/vacuum.c | 5 +++
src/include/commands/progress.h | 3 ++
src/test/regress/expected/rules.out | 4 +-
6 files changed, 67 insertions(+), 25 deletions(-)
diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml
index 3513e127b7..dca47f6a48 100644
--- a/doc/src/sgml/monitoring.sgml
+++ b/doc/src/sgml/monitoring.sgml
@@ -6231,6 +6231,27 @@ SELECT pg_stat_get_backend_pid(s.backendid) AS pid,
is <literal>rebuilding index</literal>.
</para></entry>
</row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>total_partitions</structfield> <type>bigint</type>
+ </para>
+ When clustering a partitioned table, this column is set to the total
+ number of partitions to be clustered.
+ <para>
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>done_partitions</structfield> <type>bigint</type>
+ </para>
+ <para>
+ When clustering a partitioned table, this column is set to the number
+ of partitions which have been clustered.
+ </para></entry>
+ </row>
+
</tbody>
</tgroup>
</table>
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fc94a73a54..21c27f0437 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1070,7 +1070,9 @@ CREATE VIEW pg_stat_progress_cluster AS
S.param5 AS heap_tuples_written,
S.param6 AS heap_blks_total,
S.param7 AS heap_blks_scanned,
- S.param8 AS index_rebuild_count
+ S.param8 AS index_rebuild_count,
+ S.param18 AS partitions_total,
+ S.param19 AS partitions_done
FROM pg_stat_get_progress_info('CLUSTER') AS S
LEFT JOIN pg_database D ON S.datid = D.oid;
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 0c08ac56dc..7d8d1d8a69 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
static List *get_tables_to_cluster(MemoryContext cluster_context);
static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
/*---------------------------------------------------------------------------
@@ -188,16 +188,26 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
/* close relation, keep lock till commit */
table_close(rel, ShareUpdateExclusiveLock);
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
{
/* Do the job. */
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_CLUSTER);
cluster_rel(tableOid, indexOid, ¶ms);
+ pgstat_progress_end_command();
}
else
{
List *rvs;
MemoryContext cluster_context;
+ int64 progress_val[2];
+ const int progress_index[2] = {
+ PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+ };
+
/* Refuse to hold strong locks in a user transaction */
PreventInTransactionBlock(isTopLevel, "CLUSTER");
@@ -206,7 +216,15 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
ALLOCSET_DEFAULT_SIZES);
rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
- cluster_multiple_rels(rvs, params.options);
+
+ /* Report command and number of partitions */
+ progress_val[0] = PROGRESS_CLUSTER_COMMAND_CLUSTER;
+ /* XXX: rvs includes the parent, which is not a "partition" */
+ progress_val[1] = list_length(rvs);
+
+ pgstat_progress_update_multi_param(2, progress_index, progress_val);
+ cluster_multiple_rels(rvs, params.options, true);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -245,7 +263,11 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
* cluster_context.
*/
rvs = get_tables_to_cluster(cluster_context);
- cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_CLUSTER);
+ cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -282,14 +304,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* Check for user-requested abort. */
CHECK_FOR_INTERRUPTS();
- pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
- if (OidIsValid(indexOid))
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_CLUSTER);
- else
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
/*
* We grab exclusive access to the target rel and index for the duration
* of the transaction. (This is redundant for the single-transaction
@@ -300,10 +314,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* If the table has gone away, we can skip processing it */
if (!OldHeap)
- {
- pgstat_progress_end_command();
return;
- }
/*
* Since we may open a new transaction for each relation, we have to check
@@ -319,7 +330,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!pg_class_ownercheck(tableOid, GetUserId()))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -334,7 +344,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (RELATION_IS_OTHER_TEMP(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -346,7 +355,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -357,7 +365,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!get_index_isclustered(indexOid))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
}
@@ -416,7 +423,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!RelationIsPopulated(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -424,7 +430,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -440,8 +445,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
rebuild_relation(OldHeap, indexOid, verbose);
/* NB: rebuild_relation does table_close() on OldHeap */
-
- pgstat_progress_end_command();
}
/*
@@ -1632,9 +1635,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
/* Cluster each relation in a separate transaction */
static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
{
ListCell *lc;
+ off_t childnum = 0;
/* Commit to get out of starting transaction */
PopActiveSnapshot();
@@ -1657,6 +1661,11 @@ cluster_multiple_rels(List *rvs, int options)
cluster_rel(rvtc->tableOid, rvtc->indexOid,
&cluster_params);
+ if (ispartitioned)
+ pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+ ++childnum);
+
+
PopActiveSnapshot();
CommitTransactionCommand();
}
diff --git a/src/backend/commands/vacuum.c b/src/backend/commands/vacuum.c
index c064352e23..05d52858b1 100644
--- a/src/backend/commands/vacuum.c
+++ b/src/backend/commands/vacuum.c
@@ -37,6 +37,7 @@
#include "catalog/pg_namespace.h"
#include "commands/cluster.h"
#include "commands/defrem.h"
+#include "commands/progress.h"
#include "commands/vacuum.h"
#include "miscadmin.h"
#include "nodes/makefuncs.h"
@@ -1937,7 +1938,11 @@ vacuum_rel(Oid relid, RangeVar *relation, VacuumParams *params)
cluster_params.options |= CLUOPT_VERBOSE;
/* VACUUM FULL is now a variant of CLUSTER; see cluster.c */
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, relid);
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
cluster_rel(relid, InvalidOid, &cluster_params);
+ pgstat_progress_end_command();
}
else
table_relation_vacuum(onerel, params, vac_strategy);
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
#define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS 5
#define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED 6
#define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT 7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL 17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE 18
/* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
#define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP 1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b1c9b7bdfe..c888415235 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1943,7 +1943,9 @@ pg_stat_progress_cluster| SELECT s.pid,
s.param5 AS heap_tuples_written,
s.param6 AS heap_blks_total,
s.param7 AS heap_blks_scanned,
- s.param8 AS index_rebuild_count
+ s.param8 AS index_rebuild_count,
+ s.param18 AS partitions_total,
+ s.param19 AS partitions_done
FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
LEFT JOIN pg_database d ON ((s.datid = d.oid)));
pg_stat_progress_copy| SELECT s.pid,
--
2.17.0
--fmvA4kSBHQVZhkR6
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v9-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"
^ permalink raw reply [nested|flat] 73+ messages in thread
* [PATCH v9 3/8] f! progress reporting..
@ 2021-02-09 02:45 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 73+ messages in thread
From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)
This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.
This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
doc/src/sgml/monitoring.sgml | 21 +++++++++++
src/backend/catalog/system_views.sql | 4 +-
src/backend/commands/cluster.c | 55 ++++++++++++++++------------
src/backend/commands/vacuum.c | 5 +++
src/include/commands/progress.h | 3 ++
src/test/regress/expected/rules.out | 4 +-
6 files changed, 67 insertions(+), 25 deletions(-)
diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml
index 3513e127b7..dca47f6a48 100644
--- a/doc/src/sgml/monitoring.sgml
+++ b/doc/src/sgml/monitoring.sgml
@@ -6231,6 +6231,27 @@ SELECT pg_stat_get_backend_pid(s.backendid) AS pid,
is <literal>rebuilding index</literal>.
</para></entry>
</row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>total_partitions</structfield> <type>bigint</type>
+ </para>
+ When clustering a partitioned table, this column is set to the total
+ number of partitions to be clustered.
+ <para>
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>done_partitions</structfield> <type>bigint</type>
+ </para>
+ <para>
+ When clustering a partitioned table, this column is set to the number
+ of partitions which have been clustered.
+ </para></entry>
+ </row>
+
</tbody>
</tgroup>
</table>
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fc94a73a54..21c27f0437 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1070,7 +1070,9 @@ CREATE VIEW pg_stat_progress_cluster AS
S.param5 AS heap_tuples_written,
S.param6 AS heap_blks_total,
S.param7 AS heap_blks_scanned,
- S.param8 AS index_rebuild_count
+ S.param8 AS index_rebuild_count,
+ S.param18 AS partitions_total,
+ S.param19 AS partitions_done
FROM pg_stat_get_progress_info('CLUSTER') AS S
LEFT JOIN pg_database D ON S.datid = D.oid;
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 0c08ac56dc..7d8d1d8a69 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
static List *get_tables_to_cluster(MemoryContext cluster_context);
static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
/*---------------------------------------------------------------------------
@@ -188,16 +188,26 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
/* close relation, keep lock till commit */
table_close(rel, ShareUpdateExclusiveLock);
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
{
/* Do the job. */
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_CLUSTER);
cluster_rel(tableOid, indexOid, ¶ms);
+ pgstat_progress_end_command();
}
else
{
List *rvs;
MemoryContext cluster_context;
+ int64 progress_val[2];
+ const int progress_index[2] = {
+ PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+ };
+
/* Refuse to hold strong locks in a user transaction */
PreventInTransactionBlock(isTopLevel, "CLUSTER");
@@ -206,7 +216,15 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
ALLOCSET_DEFAULT_SIZES);
rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
- cluster_multiple_rels(rvs, params.options);
+
+ /* Report command and number of partitions */
+ progress_val[0] = PROGRESS_CLUSTER_COMMAND_CLUSTER;
+ /* XXX: rvs includes the parent, which is not a "partition" */
+ progress_val[1] = list_length(rvs);
+
+ pgstat_progress_update_multi_param(2, progress_index, progress_val);
+ cluster_multiple_rels(rvs, params.options, true);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -245,7 +263,11 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
* cluster_context.
*/
rvs = get_tables_to_cluster(cluster_context);
- cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_CLUSTER);
+ cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -282,14 +304,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* Check for user-requested abort. */
CHECK_FOR_INTERRUPTS();
- pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
- if (OidIsValid(indexOid))
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_CLUSTER);
- else
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
/*
* We grab exclusive access to the target rel and index for the duration
* of the transaction. (This is redundant for the single-transaction
@@ -300,10 +314,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* If the table has gone away, we can skip processing it */
if (!OldHeap)
- {
- pgstat_progress_end_command();
return;
- }
/*
* Since we may open a new transaction for each relation, we have to check
@@ -319,7 +330,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!pg_class_ownercheck(tableOid, GetUserId()))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -334,7 +344,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (RELATION_IS_OTHER_TEMP(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -346,7 +355,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -357,7 +365,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!get_index_isclustered(indexOid))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
}
@@ -416,7 +423,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!RelationIsPopulated(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -424,7 +430,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -440,8 +445,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
rebuild_relation(OldHeap, indexOid, verbose);
/* NB: rebuild_relation does table_close() on OldHeap */
-
- pgstat_progress_end_command();
}
/*
@@ -1632,9 +1635,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
/* Cluster each relation in a separate transaction */
static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
{
ListCell *lc;
+ off_t childnum = 0;
/* Commit to get out of starting transaction */
PopActiveSnapshot();
@@ -1657,6 +1661,11 @@ cluster_multiple_rels(List *rvs, int options)
cluster_rel(rvtc->tableOid, rvtc->indexOid,
&cluster_params);
+ if (ispartitioned)
+ pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+ ++childnum);
+
+
PopActiveSnapshot();
CommitTransactionCommand();
}
diff --git a/src/backend/commands/vacuum.c b/src/backend/commands/vacuum.c
index c064352e23..05d52858b1 100644
--- a/src/backend/commands/vacuum.c
+++ b/src/backend/commands/vacuum.c
@@ -37,6 +37,7 @@
#include "catalog/pg_namespace.h"
#include "commands/cluster.h"
#include "commands/defrem.h"
+#include "commands/progress.h"
#include "commands/vacuum.h"
#include "miscadmin.h"
#include "nodes/makefuncs.h"
@@ -1937,7 +1938,11 @@ vacuum_rel(Oid relid, RangeVar *relation, VacuumParams *params)
cluster_params.options |= CLUOPT_VERBOSE;
/* VACUUM FULL is now a variant of CLUSTER; see cluster.c */
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, relid);
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
cluster_rel(relid, InvalidOid, &cluster_params);
+ pgstat_progress_end_command();
}
else
table_relation_vacuum(onerel, params, vac_strategy);
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
#define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS 5
#define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED 6
#define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT 7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL 17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE 18
/* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
#define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP 1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b1c9b7bdfe..c888415235 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1943,7 +1943,9 @@ pg_stat_progress_cluster| SELECT s.pid,
s.param5 AS heap_tuples_written,
s.param6 AS heap_blks_total,
s.param7 AS heap_blks_scanned,
- s.param8 AS index_rebuild_count
+ s.param8 AS index_rebuild_count,
+ s.param18 AS partitions_total,
+ s.param19 AS partitions_done
FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
LEFT JOIN pg_database d ON ((s.datid = d.oid)));
pg_stat_progress_copy| SELECT s.pid,
--
2.17.0
--fmvA4kSBHQVZhkR6
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v9-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"
^ permalink raw reply [nested|flat] 73+ messages in thread
* [PATCH v9 3/8] f! progress reporting..
@ 2021-02-09 02:45 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 73+ messages in thread
From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)
This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.
This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
doc/src/sgml/monitoring.sgml | 21 +++++++++++
src/backend/catalog/system_views.sql | 4 +-
src/backend/commands/cluster.c | 55 ++++++++++++++++------------
src/backend/commands/vacuum.c | 5 +++
src/include/commands/progress.h | 3 ++
src/test/regress/expected/rules.out | 4 +-
6 files changed, 67 insertions(+), 25 deletions(-)
diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml
index 3513e127b7..dca47f6a48 100644
--- a/doc/src/sgml/monitoring.sgml
+++ b/doc/src/sgml/monitoring.sgml
@@ -6231,6 +6231,27 @@ SELECT pg_stat_get_backend_pid(s.backendid) AS pid,
is <literal>rebuilding index</literal>.
</para></entry>
</row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>total_partitions</structfield> <type>bigint</type>
+ </para>
+ When clustering a partitioned table, this column is set to the total
+ number of partitions to be clustered.
+ <para>
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>done_partitions</structfield> <type>bigint</type>
+ </para>
+ <para>
+ When clustering a partitioned table, this column is set to the number
+ of partitions which have been clustered.
+ </para></entry>
+ </row>
+
</tbody>
</tgroup>
</table>
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fc94a73a54..21c27f0437 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1070,7 +1070,9 @@ CREATE VIEW pg_stat_progress_cluster AS
S.param5 AS heap_tuples_written,
S.param6 AS heap_blks_total,
S.param7 AS heap_blks_scanned,
- S.param8 AS index_rebuild_count
+ S.param8 AS index_rebuild_count,
+ S.param18 AS partitions_total,
+ S.param19 AS partitions_done
FROM pg_stat_get_progress_info('CLUSTER') AS S
LEFT JOIN pg_database D ON S.datid = D.oid;
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 0c08ac56dc..7d8d1d8a69 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
static List *get_tables_to_cluster(MemoryContext cluster_context);
static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
/*---------------------------------------------------------------------------
@@ -188,16 +188,26 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
/* close relation, keep lock till commit */
table_close(rel, ShareUpdateExclusiveLock);
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
{
/* Do the job. */
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_CLUSTER);
cluster_rel(tableOid, indexOid, ¶ms);
+ pgstat_progress_end_command();
}
else
{
List *rvs;
MemoryContext cluster_context;
+ int64 progress_val[2];
+ const int progress_index[2] = {
+ PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+ };
+
/* Refuse to hold strong locks in a user transaction */
PreventInTransactionBlock(isTopLevel, "CLUSTER");
@@ -206,7 +216,15 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
ALLOCSET_DEFAULT_SIZES);
rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
- cluster_multiple_rels(rvs, params.options);
+
+ /* Report command and number of partitions */
+ progress_val[0] = PROGRESS_CLUSTER_COMMAND_CLUSTER;
+ /* XXX: rvs includes the parent, which is not a "partition" */
+ progress_val[1] = list_length(rvs);
+
+ pgstat_progress_update_multi_param(2, progress_index, progress_val);
+ cluster_multiple_rels(rvs, params.options, true);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -245,7 +263,11 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
* cluster_context.
*/
rvs = get_tables_to_cluster(cluster_context);
- cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_CLUSTER);
+ cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -282,14 +304,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* Check for user-requested abort. */
CHECK_FOR_INTERRUPTS();
- pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
- if (OidIsValid(indexOid))
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_CLUSTER);
- else
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
/*
* We grab exclusive access to the target rel and index for the duration
* of the transaction. (This is redundant for the single-transaction
@@ -300,10 +314,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* If the table has gone away, we can skip processing it */
if (!OldHeap)
- {
- pgstat_progress_end_command();
return;
- }
/*
* Since we may open a new transaction for each relation, we have to check
@@ -319,7 +330,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!pg_class_ownercheck(tableOid, GetUserId()))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -334,7 +344,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (RELATION_IS_OTHER_TEMP(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -346,7 +355,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -357,7 +365,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!get_index_isclustered(indexOid))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
}
@@ -416,7 +423,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!RelationIsPopulated(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -424,7 +430,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -440,8 +445,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
rebuild_relation(OldHeap, indexOid, verbose);
/* NB: rebuild_relation does table_close() on OldHeap */
-
- pgstat_progress_end_command();
}
/*
@@ -1632,9 +1635,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
/* Cluster each relation in a separate transaction */
static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
{
ListCell *lc;
+ off_t childnum = 0;
/* Commit to get out of starting transaction */
PopActiveSnapshot();
@@ -1657,6 +1661,11 @@ cluster_multiple_rels(List *rvs, int options)
cluster_rel(rvtc->tableOid, rvtc->indexOid,
&cluster_params);
+ if (ispartitioned)
+ pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+ ++childnum);
+
+
PopActiveSnapshot();
CommitTransactionCommand();
}
diff --git a/src/backend/commands/vacuum.c b/src/backend/commands/vacuum.c
index c064352e23..05d52858b1 100644
--- a/src/backend/commands/vacuum.c
+++ b/src/backend/commands/vacuum.c
@@ -37,6 +37,7 @@
#include "catalog/pg_namespace.h"
#include "commands/cluster.h"
#include "commands/defrem.h"
+#include "commands/progress.h"
#include "commands/vacuum.h"
#include "miscadmin.h"
#include "nodes/makefuncs.h"
@@ -1937,7 +1938,11 @@ vacuum_rel(Oid relid, RangeVar *relation, VacuumParams *params)
cluster_params.options |= CLUOPT_VERBOSE;
/* VACUUM FULL is now a variant of CLUSTER; see cluster.c */
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, relid);
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
cluster_rel(relid, InvalidOid, &cluster_params);
+ pgstat_progress_end_command();
}
else
table_relation_vacuum(onerel, params, vac_strategy);
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
#define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS 5
#define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED 6
#define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT 7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL 17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE 18
/* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
#define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP 1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b1c9b7bdfe..c888415235 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1943,7 +1943,9 @@ pg_stat_progress_cluster| SELECT s.pid,
s.param5 AS heap_tuples_written,
s.param6 AS heap_blks_total,
s.param7 AS heap_blks_scanned,
- s.param8 AS index_rebuild_count
+ s.param8 AS index_rebuild_count,
+ s.param18 AS partitions_total,
+ s.param19 AS partitions_done
FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
LEFT JOIN pg_database d ON ((s.datid = d.oid)));
pg_stat_progress_copy| SELECT s.pid,
--
2.17.0
--fmvA4kSBHQVZhkR6
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v9-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"
^ permalink raw reply [nested|flat] 73+ messages in thread
* [PATCH v9 3/8] f! progress reporting..
@ 2021-02-09 02:45 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 73+ messages in thread
From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)
This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.
This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
doc/src/sgml/monitoring.sgml | 21 +++++++++++
src/backend/catalog/system_views.sql | 4 +-
src/backend/commands/cluster.c | 55 ++++++++++++++++------------
src/backend/commands/vacuum.c | 5 +++
src/include/commands/progress.h | 3 ++
src/test/regress/expected/rules.out | 4 +-
6 files changed, 67 insertions(+), 25 deletions(-)
diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml
index 3513e127b7..dca47f6a48 100644
--- a/doc/src/sgml/monitoring.sgml
+++ b/doc/src/sgml/monitoring.sgml
@@ -6231,6 +6231,27 @@ SELECT pg_stat_get_backend_pid(s.backendid) AS pid,
is <literal>rebuilding index</literal>.
</para></entry>
</row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>total_partitions</structfield> <type>bigint</type>
+ </para>
+ When clustering a partitioned table, this column is set to the total
+ number of partitions to be clustered.
+ <para>
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>done_partitions</structfield> <type>bigint</type>
+ </para>
+ <para>
+ When clustering a partitioned table, this column is set to the number
+ of partitions which have been clustered.
+ </para></entry>
+ </row>
+
</tbody>
</tgroup>
</table>
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fc94a73a54..21c27f0437 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1070,7 +1070,9 @@ CREATE VIEW pg_stat_progress_cluster AS
S.param5 AS heap_tuples_written,
S.param6 AS heap_blks_total,
S.param7 AS heap_blks_scanned,
- S.param8 AS index_rebuild_count
+ S.param8 AS index_rebuild_count,
+ S.param18 AS partitions_total,
+ S.param19 AS partitions_done
FROM pg_stat_get_progress_info('CLUSTER') AS S
LEFT JOIN pg_database D ON S.datid = D.oid;
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 0c08ac56dc..7d8d1d8a69 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
static List *get_tables_to_cluster(MemoryContext cluster_context);
static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
/*---------------------------------------------------------------------------
@@ -188,16 +188,26 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
/* close relation, keep lock till commit */
table_close(rel, ShareUpdateExclusiveLock);
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
{
/* Do the job. */
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_CLUSTER);
cluster_rel(tableOid, indexOid, ¶ms);
+ pgstat_progress_end_command();
}
else
{
List *rvs;
MemoryContext cluster_context;
+ int64 progress_val[2];
+ const int progress_index[2] = {
+ PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+ };
+
/* Refuse to hold strong locks in a user transaction */
PreventInTransactionBlock(isTopLevel, "CLUSTER");
@@ -206,7 +216,15 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
ALLOCSET_DEFAULT_SIZES);
rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
- cluster_multiple_rels(rvs, params.options);
+
+ /* Report command and number of partitions */
+ progress_val[0] = PROGRESS_CLUSTER_COMMAND_CLUSTER;
+ /* XXX: rvs includes the parent, which is not a "partition" */
+ progress_val[1] = list_length(rvs);
+
+ pgstat_progress_update_multi_param(2, progress_index, progress_val);
+ cluster_multiple_rels(rvs, params.options, true);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -245,7 +263,11 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
* cluster_context.
*/
rvs = get_tables_to_cluster(cluster_context);
- cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_CLUSTER);
+ cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -282,14 +304,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* Check for user-requested abort. */
CHECK_FOR_INTERRUPTS();
- pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
- if (OidIsValid(indexOid))
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_CLUSTER);
- else
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
/*
* We grab exclusive access to the target rel and index for the duration
* of the transaction. (This is redundant for the single-transaction
@@ -300,10 +314,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* If the table has gone away, we can skip processing it */
if (!OldHeap)
- {
- pgstat_progress_end_command();
return;
- }
/*
* Since we may open a new transaction for each relation, we have to check
@@ -319,7 +330,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!pg_class_ownercheck(tableOid, GetUserId()))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -334,7 +344,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (RELATION_IS_OTHER_TEMP(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -346,7 +355,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -357,7 +365,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!get_index_isclustered(indexOid))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
}
@@ -416,7 +423,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!RelationIsPopulated(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -424,7 +430,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -440,8 +445,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
rebuild_relation(OldHeap, indexOid, verbose);
/* NB: rebuild_relation does table_close() on OldHeap */
-
- pgstat_progress_end_command();
}
/*
@@ -1632,9 +1635,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
/* Cluster each relation in a separate transaction */
static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
{
ListCell *lc;
+ off_t childnum = 0;
/* Commit to get out of starting transaction */
PopActiveSnapshot();
@@ -1657,6 +1661,11 @@ cluster_multiple_rels(List *rvs, int options)
cluster_rel(rvtc->tableOid, rvtc->indexOid,
&cluster_params);
+ if (ispartitioned)
+ pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+ ++childnum);
+
+
PopActiveSnapshot();
CommitTransactionCommand();
}
diff --git a/src/backend/commands/vacuum.c b/src/backend/commands/vacuum.c
index c064352e23..05d52858b1 100644
--- a/src/backend/commands/vacuum.c
+++ b/src/backend/commands/vacuum.c
@@ -37,6 +37,7 @@
#include "catalog/pg_namespace.h"
#include "commands/cluster.h"
#include "commands/defrem.h"
+#include "commands/progress.h"
#include "commands/vacuum.h"
#include "miscadmin.h"
#include "nodes/makefuncs.h"
@@ -1937,7 +1938,11 @@ vacuum_rel(Oid relid, RangeVar *relation, VacuumParams *params)
cluster_params.options |= CLUOPT_VERBOSE;
/* VACUUM FULL is now a variant of CLUSTER; see cluster.c */
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, relid);
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
cluster_rel(relid, InvalidOid, &cluster_params);
+ pgstat_progress_end_command();
}
else
table_relation_vacuum(onerel, params, vac_strategy);
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
#define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS 5
#define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED 6
#define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT 7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL 17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE 18
/* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
#define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP 1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b1c9b7bdfe..c888415235 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1943,7 +1943,9 @@ pg_stat_progress_cluster| SELECT s.pid,
s.param5 AS heap_tuples_written,
s.param6 AS heap_blks_total,
s.param7 AS heap_blks_scanned,
- s.param8 AS index_rebuild_count
+ s.param8 AS index_rebuild_count,
+ s.param18 AS partitions_total,
+ s.param19 AS partitions_done
FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
LEFT JOIN pg_database d ON ((s.datid = d.oid)));
pg_stat_progress_copy| SELECT s.pid,
--
2.17.0
--fmvA4kSBHQVZhkR6
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v9-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"
^ permalink raw reply [nested|flat] 73+ messages in thread
* [PATCH v8 3/8] f! progress reporting..
@ 2021-02-09 02:45 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 73+ messages in thread
From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)
This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.
This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
src/backend/catalog/system_views.sql | 4 +-
src/backend/commands/cluster.c | 64 +++++++++++++++++-----------
src/include/commands/progress.h | 3 ++
src/test/regress/expected/rules.out | 4 +-
4 files changed, 47 insertions(+), 28 deletions(-)
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fa58afd9d7..9a3cb5f935 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1069,7 +1069,9 @@ CREATE VIEW pg_stat_progress_cluster AS
S.param5 AS heap_tuples_written,
S.param6 AS heap_blks_total,
S.param7 AS heap_blks_scanned,
- S.param8 AS index_rebuild_count
+ S.param8 AS index_rebuild_count,
+ S.param18 AS partitions_total,
+ S.param19 AS partitions_done
FROM pg_stat_get_progress_info('CLUSTER') AS S
LEFT JOIN pg_database D ON S.datid = D.oid;
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 9b6673867c..cb4fc350c6 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
static List *get_tables_to_cluster(MemoryContext cluster_context);
static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
/*---------------------------------------------------------------------------
@@ -188,16 +188,28 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
/* close relation, keep lock till commit */
table_close(rel, ShareUpdateExclusiveLock);
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
{
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ OidIsValid(indexOid) ? PROGRESS_CLUSTER_COMMAND_CLUSTER :
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
+
/* Do the job. */
cluster_rel(tableOid, indexOid, ¶ms);
+ pgstat_progress_end_command();
}
else
{
List *rvs;
MemoryContext cluster_context;
+ int64 progress_val[2];
+ const int progress_index[] = {
+ PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+ };
+
/* Refuse to hold strong locks in a user transaction */
PreventInTransactionBlock(isTopLevel, "CLUSTER");
@@ -206,7 +218,20 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
ALLOCSET_DEFAULT_SIZES);
rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
- cluster_multiple_rels(rvs, params.options);
+
+ /* Report vacuum full or cluster and number of partitions */
+ progress_val[0] = OidIsValid(indexOid) ?
+ PROGRESS_CLUSTER_COMMAND_CLUSTER :
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL;
+ progress_val[1] = list_length(rvs);
+// This is currently showing the total number of *tables*, including the
+// parent, and intermediate partitions, so the column should be renamed, or
+// we should count the number of leaf partitions
+
+ pgstat_progress_update_multi_param(2, progress_index, progress_val);
+
+ cluster_multiple_rels(rvs, params.options, true);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -245,7 +270,7 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
* cluster_context.
*/
rvs = get_tables_to_cluster(cluster_context);
- cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+ cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -282,14 +307,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* Check for user-requested abort. */
CHECK_FOR_INTERRUPTS();
- pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
- if (OidIsValid(indexOid))
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_CLUSTER);
- else
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
/*
* We grab exclusive access to the target rel and index for the duration
* of the transaction. (This is redundant for the single-transaction
@@ -300,10 +317,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* If the table has gone away, we can skip processing it */
if (!OldHeap)
- {
- pgstat_progress_end_command();
return;
- }
/*
* Since we may open a new transaction for each relation, we have to check
@@ -319,7 +333,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!pg_class_ownercheck(tableOid, GetUserId()))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -334,7 +347,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (RELATION_IS_OTHER_TEMP(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -346,7 +358,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -357,7 +368,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!get_index_isclustered(indexOid))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
}
@@ -416,7 +426,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!RelationIsPopulated(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -424,7 +433,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -440,8 +448,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
rebuild_relation(OldHeap, indexOid, verbose);
/* NB: rebuild_relation does table_close() on OldHeap */
-
- pgstat_progress_end_command();
}
/*
@@ -1366,7 +1372,7 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap,
ReindexParams reindex_params = {0};
int i;
- /* Report that we are now swapping relation files */
+ /* Report that we are now swapping relation files XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_SWAP_REL_FILES);
@@ -1417,13 +1423,13 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap,
else if (newrelpersistence == RELPERSISTENCE_PERMANENT)
reindex_flags |= REINDEX_REL_FORCE_INDEXES_PERMANENT;
- /* Report that we are now reindexing relations */
+ /* Report that we are now reindexing relations XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_REBUILD_INDEX);
reindex_relation(OIDOldHeap, reindex_flags, &reindex_params);
- /* Report that we are now doing clean up */
+ /* Report that we are now doing clean up XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_FINAL_CLEANUP);
@@ -1633,9 +1639,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
/* Cluster each relation in a separate transaction */
static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
{
ListCell *lc;
+ off_t childnum = 0;
/* Commit to get out of starting transaction */
PopActiveSnapshot();
@@ -1658,6 +1665,11 @@ cluster_multiple_rels(List *rvs, int options)
cluster_rel(rvtc->tableOid, rvtc->indexOid,
&cluster_params);
+ if (ispartitioned)
+ pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+ childnum++);
+
+
PopActiveSnapshot();
CommitTransactionCommand();
}
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
#define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS 5
#define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED 6
#define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT 7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL 17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE 18
/* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
#define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP 1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b632d9f2ea..d1fea532bc 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1941,7 +1941,9 @@ pg_stat_progress_cluster| SELECT s.pid,
s.param5 AS heap_tuples_written,
s.param6 AS heap_blks_total,
s.param7 AS heap_blks_scanned,
- s.param8 AS index_rebuild_count
+ s.param8 AS index_rebuild_count,
+ s.param18 AS partitions_total,
+ s.param19 AS partitions_done
FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
LEFT JOIN pg_database d ON ((s.datid = d.oid)));
pg_stat_progress_copy| SELECT s.pid,
--
2.17.0
--O5XBE6gyVG5Rl6Rj
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v8-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"
^ permalink raw reply [nested|flat] 73+ messages in thread
* [PATCH v8 3/8] f! progress reporting..
@ 2021-02-09 02:45 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 73+ messages in thread
From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)
This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.
This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
src/backend/catalog/system_views.sql | 4 +-
src/backend/commands/cluster.c | 64 +++++++++++++++++-----------
src/include/commands/progress.h | 3 ++
src/test/regress/expected/rules.out | 4 +-
4 files changed, 47 insertions(+), 28 deletions(-)
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fa58afd9d7..9a3cb5f935 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1069,7 +1069,9 @@ CREATE VIEW pg_stat_progress_cluster AS
S.param5 AS heap_tuples_written,
S.param6 AS heap_blks_total,
S.param7 AS heap_blks_scanned,
- S.param8 AS index_rebuild_count
+ S.param8 AS index_rebuild_count,
+ S.param18 AS partitions_total,
+ S.param19 AS partitions_done
FROM pg_stat_get_progress_info('CLUSTER') AS S
LEFT JOIN pg_database D ON S.datid = D.oid;
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 9b6673867c..cb4fc350c6 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
static List *get_tables_to_cluster(MemoryContext cluster_context);
static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
/*---------------------------------------------------------------------------
@@ -188,16 +188,28 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
/* close relation, keep lock till commit */
table_close(rel, ShareUpdateExclusiveLock);
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
{
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ OidIsValid(indexOid) ? PROGRESS_CLUSTER_COMMAND_CLUSTER :
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
+
/* Do the job. */
cluster_rel(tableOid, indexOid, ¶ms);
+ pgstat_progress_end_command();
}
else
{
List *rvs;
MemoryContext cluster_context;
+ int64 progress_val[2];
+ const int progress_index[] = {
+ PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+ };
+
/* Refuse to hold strong locks in a user transaction */
PreventInTransactionBlock(isTopLevel, "CLUSTER");
@@ -206,7 +218,20 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
ALLOCSET_DEFAULT_SIZES);
rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
- cluster_multiple_rels(rvs, params.options);
+
+ /* Report vacuum full or cluster and number of partitions */
+ progress_val[0] = OidIsValid(indexOid) ?
+ PROGRESS_CLUSTER_COMMAND_CLUSTER :
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL;
+ progress_val[1] = list_length(rvs);
+// This is currently showing the total number of *tables*, including the
+// parent, and intermediate partitions, so the column should be renamed, or
+// we should count the number of leaf partitions
+
+ pgstat_progress_update_multi_param(2, progress_index, progress_val);
+
+ cluster_multiple_rels(rvs, params.options, true);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -245,7 +270,7 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
* cluster_context.
*/
rvs = get_tables_to_cluster(cluster_context);
- cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+ cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -282,14 +307,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* Check for user-requested abort. */
CHECK_FOR_INTERRUPTS();
- pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
- if (OidIsValid(indexOid))
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_CLUSTER);
- else
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
/*
* We grab exclusive access to the target rel and index for the duration
* of the transaction. (This is redundant for the single-transaction
@@ -300,10 +317,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* If the table has gone away, we can skip processing it */
if (!OldHeap)
- {
- pgstat_progress_end_command();
return;
- }
/*
* Since we may open a new transaction for each relation, we have to check
@@ -319,7 +333,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!pg_class_ownercheck(tableOid, GetUserId()))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -334,7 +347,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (RELATION_IS_OTHER_TEMP(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -346,7 +358,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -357,7 +368,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!get_index_isclustered(indexOid))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
}
@@ -416,7 +426,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!RelationIsPopulated(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -424,7 +433,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -440,8 +448,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
rebuild_relation(OldHeap, indexOid, verbose);
/* NB: rebuild_relation does table_close() on OldHeap */
-
- pgstat_progress_end_command();
}
/*
@@ -1366,7 +1372,7 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap,
ReindexParams reindex_params = {0};
int i;
- /* Report that we are now swapping relation files */
+ /* Report that we are now swapping relation files XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_SWAP_REL_FILES);
@@ -1417,13 +1423,13 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap,
else if (newrelpersistence == RELPERSISTENCE_PERMANENT)
reindex_flags |= REINDEX_REL_FORCE_INDEXES_PERMANENT;
- /* Report that we are now reindexing relations */
+ /* Report that we are now reindexing relations XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_REBUILD_INDEX);
reindex_relation(OIDOldHeap, reindex_flags, &reindex_params);
- /* Report that we are now doing clean up */
+ /* Report that we are now doing clean up XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_FINAL_CLEANUP);
@@ -1633,9 +1639,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
/* Cluster each relation in a separate transaction */
static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
{
ListCell *lc;
+ off_t childnum = 0;
/* Commit to get out of starting transaction */
PopActiveSnapshot();
@@ -1658,6 +1665,11 @@ cluster_multiple_rels(List *rvs, int options)
cluster_rel(rvtc->tableOid, rvtc->indexOid,
&cluster_params);
+ if (ispartitioned)
+ pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+ childnum++);
+
+
PopActiveSnapshot();
CommitTransactionCommand();
}
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
#define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS 5
#define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED 6
#define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT 7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL 17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE 18
/* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
#define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP 1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b632d9f2ea..d1fea532bc 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1941,7 +1941,9 @@ pg_stat_progress_cluster| SELECT s.pid,
s.param5 AS heap_tuples_written,
s.param6 AS heap_blks_total,
s.param7 AS heap_blks_scanned,
- s.param8 AS index_rebuild_count
+ s.param8 AS index_rebuild_count,
+ s.param18 AS partitions_total,
+ s.param19 AS partitions_done
FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
LEFT JOIN pg_database d ON ((s.datid = d.oid)));
pg_stat_progress_copy| SELECT s.pid,
--
2.17.0
--O5XBE6gyVG5Rl6Rj
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v8-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"
^ permalink raw reply [nested|flat] 73+ messages in thread
* [PATCH v8 3/8] f! progress reporting..
@ 2021-02-09 02:45 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 73+ messages in thread
From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)
This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.
This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
src/backend/catalog/system_views.sql | 4 +-
src/backend/commands/cluster.c | 64 +++++++++++++++++-----------
src/include/commands/progress.h | 3 ++
src/test/regress/expected/rules.out | 4 +-
4 files changed, 47 insertions(+), 28 deletions(-)
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fa58afd9d7..9a3cb5f935 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1069,7 +1069,9 @@ CREATE VIEW pg_stat_progress_cluster AS
S.param5 AS heap_tuples_written,
S.param6 AS heap_blks_total,
S.param7 AS heap_blks_scanned,
- S.param8 AS index_rebuild_count
+ S.param8 AS index_rebuild_count,
+ S.param18 AS partitions_total,
+ S.param19 AS partitions_done
FROM pg_stat_get_progress_info('CLUSTER') AS S
LEFT JOIN pg_database D ON S.datid = D.oid;
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 9b6673867c..cb4fc350c6 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
static List *get_tables_to_cluster(MemoryContext cluster_context);
static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
/*---------------------------------------------------------------------------
@@ -188,16 +188,28 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
/* close relation, keep lock till commit */
table_close(rel, ShareUpdateExclusiveLock);
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
{
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ OidIsValid(indexOid) ? PROGRESS_CLUSTER_COMMAND_CLUSTER :
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
+
/* Do the job. */
cluster_rel(tableOid, indexOid, ¶ms);
+ pgstat_progress_end_command();
}
else
{
List *rvs;
MemoryContext cluster_context;
+ int64 progress_val[2];
+ const int progress_index[] = {
+ PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+ };
+
/* Refuse to hold strong locks in a user transaction */
PreventInTransactionBlock(isTopLevel, "CLUSTER");
@@ -206,7 +218,20 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
ALLOCSET_DEFAULT_SIZES);
rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
- cluster_multiple_rels(rvs, params.options);
+
+ /* Report vacuum full or cluster and number of partitions */
+ progress_val[0] = OidIsValid(indexOid) ?
+ PROGRESS_CLUSTER_COMMAND_CLUSTER :
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL;
+ progress_val[1] = list_length(rvs);
+// This is currently showing the total number of *tables*, including the
+// parent, and intermediate partitions, so the column should be renamed, or
+// we should count the number of leaf partitions
+
+ pgstat_progress_update_multi_param(2, progress_index, progress_val);
+
+ cluster_multiple_rels(rvs, params.options, true);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -245,7 +270,7 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
* cluster_context.
*/
rvs = get_tables_to_cluster(cluster_context);
- cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+ cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -282,14 +307,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* Check for user-requested abort. */
CHECK_FOR_INTERRUPTS();
- pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
- if (OidIsValid(indexOid))
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_CLUSTER);
- else
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
/*
* We grab exclusive access to the target rel and index for the duration
* of the transaction. (This is redundant for the single-transaction
@@ -300,10 +317,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* If the table has gone away, we can skip processing it */
if (!OldHeap)
- {
- pgstat_progress_end_command();
return;
- }
/*
* Since we may open a new transaction for each relation, we have to check
@@ -319,7 +333,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!pg_class_ownercheck(tableOid, GetUserId()))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -334,7 +347,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (RELATION_IS_OTHER_TEMP(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -346,7 +358,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -357,7 +368,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!get_index_isclustered(indexOid))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
}
@@ -416,7 +426,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!RelationIsPopulated(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -424,7 +433,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -440,8 +448,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
rebuild_relation(OldHeap, indexOid, verbose);
/* NB: rebuild_relation does table_close() on OldHeap */
-
- pgstat_progress_end_command();
}
/*
@@ -1366,7 +1372,7 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap,
ReindexParams reindex_params = {0};
int i;
- /* Report that we are now swapping relation files */
+ /* Report that we are now swapping relation files XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_SWAP_REL_FILES);
@@ -1417,13 +1423,13 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap,
else if (newrelpersistence == RELPERSISTENCE_PERMANENT)
reindex_flags |= REINDEX_REL_FORCE_INDEXES_PERMANENT;
- /* Report that we are now reindexing relations */
+ /* Report that we are now reindexing relations XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_REBUILD_INDEX);
reindex_relation(OIDOldHeap, reindex_flags, &reindex_params);
- /* Report that we are now doing clean up */
+ /* Report that we are now doing clean up XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_FINAL_CLEANUP);
@@ -1633,9 +1639,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
/* Cluster each relation in a separate transaction */
static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
{
ListCell *lc;
+ off_t childnum = 0;
/* Commit to get out of starting transaction */
PopActiveSnapshot();
@@ -1658,6 +1665,11 @@ cluster_multiple_rels(List *rvs, int options)
cluster_rel(rvtc->tableOid, rvtc->indexOid,
&cluster_params);
+ if (ispartitioned)
+ pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+ childnum++);
+
+
PopActiveSnapshot();
CommitTransactionCommand();
}
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
#define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS 5
#define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED 6
#define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT 7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL 17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE 18
/* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
#define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP 1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b632d9f2ea..d1fea532bc 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1941,7 +1941,9 @@ pg_stat_progress_cluster| SELECT s.pid,
s.param5 AS heap_tuples_written,
s.param6 AS heap_blks_total,
s.param7 AS heap_blks_scanned,
- s.param8 AS index_rebuild_count
+ s.param8 AS index_rebuild_count,
+ s.param18 AS partitions_total,
+ s.param19 AS partitions_done
FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
LEFT JOIN pg_database d ON ((s.datid = d.oid)));
pg_stat_progress_copy| SELECT s.pid,
--
2.17.0
--O5XBE6gyVG5Rl6Rj
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v8-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"
^ permalink raw reply [nested|flat] 73+ messages in thread
* [PATCH v8 3/8] f! progress reporting..
@ 2021-02-09 02:45 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 73+ messages in thread
From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)
This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.
This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
src/backend/catalog/system_views.sql | 4 +-
src/backend/commands/cluster.c | 64 +++++++++++++++++-----------
src/include/commands/progress.h | 3 ++
src/test/regress/expected/rules.out | 4 +-
4 files changed, 47 insertions(+), 28 deletions(-)
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fa58afd9d7..9a3cb5f935 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1069,7 +1069,9 @@ CREATE VIEW pg_stat_progress_cluster AS
S.param5 AS heap_tuples_written,
S.param6 AS heap_blks_total,
S.param7 AS heap_blks_scanned,
- S.param8 AS index_rebuild_count
+ S.param8 AS index_rebuild_count,
+ S.param18 AS partitions_total,
+ S.param19 AS partitions_done
FROM pg_stat_get_progress_info('CLUSTER') AS S
LEFT JOIN pg_database D ON S.datid = D.oid;
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 9b6673867c..cb4fc350c6 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
static List *get_tables_to_cluster(MemoryContext cluster_context);
static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
/*---------------------------------------------------------------------------
@@ -188,16 +188,28 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
/* close relation, keep lock till commit */
table_close(rel, ShareUpdateExclusiveLock);
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
{
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ OidIsValid(indexOid) ? PROGRESS_CLUSTER_COMMAND_CLUSTER :
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
+
/* Do the job. */
cluster_rel(tableOid, indexOid, ¶ms);
+ pgstat_progress_end_command();
}
else
{
List *rvs;
MemoryContext cluster_context;
+ int64 progress_val[2];
+ const int progress_index[] = {
+ PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+ };
+
/* Refuse to hold strong locks in a user transaction */
PreventInTransactionBlock(isTopLevel, "CLUSTER");
@@ -206,7 +218,20 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
ALLOCSET_DEFAULT_SIZES);
rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
- cluster_multiple_rels(rvs, params.options);
+
+ /* Report vacuum full or cluster and number of partitions */
+ progress_val[0] = OidIsValid(indexOid) ?
+ PROGRESS_CLUSTER_COMMAND_CLUSTER :
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL;
+ progress_val[1] = list_length(rvs);
+// This is currently showing the total number of *tables*, including the
+// parent, and intermediate partitions, so the column should be renamed, or
+// we should count the number of leaf partitions
+
+ pgstat_progress_update_multi_param(2, progress_index, progress_val);
+
+ cluster_multiple_rels(rvs, params.options, true);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -245,7 +270,7 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
* cluster_context.
*/
rvs = get_tables_to_cluster(cluster_context);
- cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+ cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -282,14 +307,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* Check for user-requested abort. */
CHECK_FOR_INTERRUPTS();
- pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
- if (OidIsValid(indexOid))
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_CLUSTER);
- else
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
/*
* We grab exclusive access to the target rel and index for the duration
* of the transaction. (This is redundant for the single-transaction
@@ -300,10 +317,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* If the table has gone away, we can skip processing it */
if (!OldHeap)
- {
- pgstat_progress_end_command();
return;
- }
/*
* Since we may open a new transaction for each relation, we have to check
@@ -319,7 +333,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!pg_class_ownercheck(tableOid, GetUserId()))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -334,7 +347,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (RELATION_IS_OTHER_TEMP(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -346,7 +358,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -357,7 +368,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!get_index_isclustered(indexOid))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
}
@@ -416,7 +426,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!RelationIsPopulated(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -424,7 +433,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -440,8 +448,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
rebuild_relation(OldHeap, indexOid, verbose);
/* NB: rebuild_relation does table_close() on OldHeap */
-
- pgstat_progress_end_command();
}
/*
@@ -1366,7 +1372,7 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap,
ReindexParams reindex_params = {0};
int i;
- /* Report that we are now swapping relation files */
+ /* Report that we are now swapping relation files XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_SWAP_REL_FILES);
@@ -1417,13 +1423,13 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap,
else if (newrelpersistence == RELPERSISTENCE_PERMANENT)
reindex_flags |= REINDEX_REL_FORCE_INDEXES_PERMANENT;
- /* Report that we are now reindexing relations */
+ /* Report that we are now reindexing relations XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_REBUILD_INDEX);
reindex_relation(OIDOldHeap, reindex_flags, &reindex_params);
- /* Report that we are now doing clean up */
+ /* Report that we are now doing clean up XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_FINAL_CLEANUP);
@@ -1633,9 +1639,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
/* Cluster each relation in a separate transaction */
static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
{
ListCell *lc;
+ off_t childnum = 0;
/* Commit to get out of starting transaction */
PopActiveSnapshot();
@@ -1658,6 +1665,11 @@ cluster_multiple_rels(List *rvs, int options)
cluster_rel(rvtc->tableOid, rvtc->indexOid,
&cluster_params);
+ if (ispartitioned)
+ pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+ childnum++);
+
+
PopActiveSnapshot();
CommitTransactionCommand();
}
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
#define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS 5
#define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED 6
#define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT 7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL 17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE 18
/* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
#define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP 1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b632d9f2ea..d1fea532bc 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1941,7 +1941,9 @@ pg_stat_progress_cluster| SELECT s.pid,
s.param5 AS heap_tuples_written,
s.param6 AS heap_blks_total,
s.param7 AS heap_blks_scanned,
- s.param8 AS index_rebuild_count
+ s.param8 AS index_rebuild_count,
+ s.param18 AS partitions_total,
+ s.param19 AS partitions_done
FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
LEFT JOIN pg_database d ON ((s.datid = d.oid)));
pg_stat_progress_copy| SELECT s.pid,
--
2.17.0
--O5XBE6gyVG5Rl6Rj
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v8-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"
^ permalink raw reply [nested|flat] 73+ messages in thread
* [PATCH v9 3/8] f! progress reporting..
@ 2021-02-09 02:45 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 73+ messages in thread
From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)
This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.
This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
doc/src/sgml/monitoring.sgml | 21 +++++++++++
src/backend/catalog/system_views.sql | 4 +-
src/backend/commands/cluster.c | 55 ++++++++++++++++------------
src/backend/commands/vacuum.c | 5 +++
src/include/commands/progress.h | 3 ++
src/test/regress/expected/rules.out | 4 +-
6 files changed, 67 insertions(+), 25 deletions(-)
diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml
index 3513e127b7..dca47f6a48 100644
--- a/doc/src/sgml/monitoring.sgml
+++ b/doc/src/sgml/monitoring.sgml
@@ -6231,6 +6231,27 @@ SELECT pg_stat_get_backend_pid(s.backendid) AS pid,
is <literal>rebuilding index</literal>.
</para></entry>
</row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>total_partitions</structfield> <type>bigint</type>
+ </para>
+ When clustering a partitioned table, this column is set to the total
+ number of partitions to be clustered.
+ <para>
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>done_partitions</structfield> <type>bigint</type>
+ </para>
+ <para>
+ When clustering a partitioned table, this column is set to the number
+ of partitions which have been clustered.
+ </para></entry>
+ </row>
+
</tbody>
</tgroup>
</table>
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fc94a73a54..21c27f0437 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1070,7 +1070,9 @@ CREATE VIEW pg_stat_progress_cluster AS
S.param5 AS heap_tuples_written,
S.param6 AS heap_blks_total,
S.param7 AS heap_blks_scanned,
- S.param8 AS index_rebuild_count
+ S.param8 AS index_rebuild_count,
+ S.param18 AS partitions_total,
+ S.param19 AS partitions_done
FROM pg_stat_get_progress_info('CLUSTER') AS S
LEFT JOIN pg_database D ON S.datid = D.oid;
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 0c08ac56dc..7d8d1d8a69 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
static List *get_tables_to_cluster(MemoryContext cluster_context);
static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
/*---------------------------------------------------------------------------
@@ -188,16 +188,26 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
/* close relation, keep lock till commit */
table_close(rel, ShareUpdateExclusiveLock);
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
{
/* Do the job. */
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_CLUSTER);
cluster_rel(tableOid, indexOid, ¶ms);
+ pgstat_progress_end_command();
}
else
{
List *rvs;
MemoryContext cluster_context;
+ int64 progress_val[2];
+ const int progress_index[2] = {
+ PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+ };
+
/* Refuse to hold strong locks in a user transaction */
PreventInTransactionBlock(isTopLevel, "CLUSTER");
@@ -206,7 +216,15 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
ALLOCSET_DEFAULT_SIZES);
rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
- cluster_multiple_rels(rvs, params.options);
+
+ /* Report command and number of partitions */
+ progress_val[0] = PROGRESS_CLUSTER_COMMAND_CLUSTER;
+ /* XXX: rvs includes the parent, which is not a "partition" */
+ progress_val[1] = list_length(rvs);
+
+ pgstat_progress_update_multi_param(2, progress_index, progress_val);
+ cluster_multiple_rels(rvs, params.options, true);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -245,7 +263,11 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
* cluster_context.
*/
rvs = get_tables_to_cluster(cluster_context);
- cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_CLUSTER);
+ cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -282,14 +304,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* Check for user-requested abort. */
CHECK_FOR_INTERRUPTS();
- pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
- if (OidIsValid(indexOid))
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_CLUSTER);
- else
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
/*
* We grab exclusive access to the target rel and index for the duration
* of the transaction. (This is redundant for the single-transaction
@@ -300,10 +314,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* If the table has gone away, we can skip processing it */
if (!OldHeap)
- {
- pgstat_progress_end_command();
return;
- }
/*
* Since we may open a new transaction for each relation, we have to check
@@ -319,7 +330,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!pg_class_ownercheck(tableOid, GetUserId()))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -334,7 +344,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (RELATION_IS_OTHER_TEMP(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -346,7 +355,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -357,7 +365,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!get_index_isclustered(indexOid))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
}
@@ -416,7 +423,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!RelationIsPopulated(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -424,7 +430,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -440,8 +445,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
rebuild_relation(OldHeap, indexOid, verbose);
/* NB: rebuild_relation does table_close() on OldHeap */
-
- pgstat_progress_end_command();
}
/*
@@ -1632,9 +1635,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
/* Cluster each relation in a separate transaction */
static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
{
ListCell *lc;
+ off_t childnum = 0;
/* Commit to get out of starting transaction */
PopActiveSnapshot();
@@ -1657,6 +1661,11 @@ cluster_multiple_rels(List *rvs, int options)
cluster_rel(rvtc->tableOid, rvtc->indexOid,
&cluster_params);
+ if (ispartitioned)
+ pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+ ++childnum);
+
+
PopActiveSnapshot();
CommitTransactionCommand();
}
diff --git a/src/backend/commands/vacuum.c b/src/backend/commands/vacuum.c
index c064352e23..05d52858b1 100644
--- a/src/backend/commands/vacuum.c
+++ b/src/backend/commands/vacuum.c
@@ -37,6 +37,7 @@
#include "catalog/pg_namespace.h"
#include "commands/cluster.h"
#include "commands/defrem.h"
+#include "commands/progress.h"
#include "commands/vacuum.h"
#include "miscadmin.h"
#include "nodes/makefuncs.h"
@@ -1937,7 +1938,11 @@ vacuum_rel(Oid relid, RangeVar *relation, VacuumParams *params)
cluster_params.options |= CLUOPT_VERBOSE;
/* VACUUM FULL is now a variant of CLUSTER; see cluster.c */
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, relid);
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
cluster_rel(relid, InvalidOid, &cluster_params);
+ pgstat_progress_end_command();
}
else
table_relation_vacuum(onerel, params, vac_strategy);
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
#define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS 5
#define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED 6
#define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT 7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL 17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE 18
/* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
#define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP 1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b1c9b7bdfe..c888415235 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1943,7 +1943,9 @@ pg_stat_progress_cluster| SELECT s.pid,
s.param5 AS heap_tuples_written,
s.param6 AS heap_blks_total,
s.param7 AS heap_blks_scanned,
- s.param8 AS index_rebuild_count
+ s.param8 AS index_rebuild_count,
+ s.param18 AS partitions_total,
+ s.param19 AS partitions_done
FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
LEFT JOIN pg_database d ON ((s.datid = d.oid)));
pg_stat_progress_copy| SELECT s.pid,
--
2.17.0
--fmvA4kSBHQVZhkR6
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v9-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"
^ permalink raw reply [nested|flat] 73+ messages in thread
* [PATCH v8 3/8] f! progress reporting..
@ 2021-02-09 02:45 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 73+ messages in thread
From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)
This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.
This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
src/backend/catalog/system_views.sql | 4 +-
src/backend/commands/cluster.c | 64 +++++++++++++++++-----------
src/include/commands/progress.h | 3 ++
src/test/regress/expected/rules.out | 4 +-
4 files changed, 47 insertions(+), 28 deletions(-)
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fa58afd9d7..9a3cb5f935 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1069,7 +1069,9 @@ CREATE VIEW pg_stat_progress_cluster AS
S.param5 AS heap_tuples_written,
S.param6 AS heap_blks_total,
S.param7 AS heap_blks_scanned,
- S.param8 AS index_rebuild_count
+ S.param8 AS index_rebuild_count,
+ S.param18 AS partitions_total,
+ S.param19 AS partitions_done
FROM pg_stat_get_progress_info('CLUSTER') AS S
LEFT JOIN pg_database D ON S.datid = D.oid;
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 9b6673867c..cb4fc350c6 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
static List *get_tables_to_cluster(MemoryContext cluster_context);
static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
/*---------------------------------------------------------------------------
@@ -188,16 +188,28 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
/* close relation, keep lock till commit */
table_close(rel, ShareUpdateExclusiveLock);
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
{
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ OidIsValid(indexOid) ? PROGRESS_CLUSTER_COMMAND_CLUSTER :
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
+
/* Do the job. */
cluster_rel(tableOid, indexOid, ¶ms);
+ pgstat_progress_end_command();
}
else
{
List *rvs;
MemoryContext cluster_context;
+ int64 progress_val[2];
+ const int progress_index[] = {
+ PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+ };
+
/* Refuse to hold strong locks in a user transaction */
PreventInTransactionBlock(isTopLevel, "CLUSTER");
@@ -206,7 +218,20 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
ALLOCSET_DEFAULT_SIZES);
rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
- cluster_multiple_rels(rvs, params.options);
+
+ /* Report vacuum full or cluster and number of partitions */
+ progress_val[0] = OidIsValid(indexOid) ?
+ PROGRESS_CLUSTER_COMMAND_CLUSTER :
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL;
+ progress_val[1] = list_length(rvs);
+// This is currently showing the total number of *tables*, including the
+// parent, and intermediate partitions, so the column should be renamed, or
+// we should count the number of leaf partitions
+
+ pgstat_progress_update_multi_param(2, progress_index, progress_val);
+
+ cluster_multiple_rels(rvs, params.options, true);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -245,7 +270,7 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
* cluster_context.
*/
rvs = get_tables_to_cluster(cluster_context);
- cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+ cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -282,14 +307,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* Check for user-requested abort. */
CHECK_FOR_INTERRUPTS();
- pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
- if (OidIsValid(indexOid))
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_CLUSTER);
- else
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
/*
* We grab exclusive access to the target rel and index for the duration
* of the transaction. (This is redundant for the single-transaction
@@ -300,10 +317,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* If the table has gone away, we can skip processing it */
if (!OldHeap)
- {
- pgstat_progress_end_command();
return;
- }
/*
* Since we may open a new transaction for each relation, we have to check
@@ -319,7 +333,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!pg_class_ownercheck(tableOid, GetUserId()))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -334,7 +347,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (RELATION_IS_OTHER_TEMP(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -346,7 +358,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -357,7 +368,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!get_index_isclustered(indexOid))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
}
@@ -416,7 +426,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!RelationIsPopulated(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -424,7 +433,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -440,8 +448,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
rebuild_relation(OldHeap, indexOid, verbose);
/* NB: rebuild_relation does table_close() on OldHeap */
-
- pgstat_progress_end_command();
}
/*
@@ -1366,7 +1372,7 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap,
ReindexParams reindex_params = {0};
int i;
- /* Report that we are now swapping relation files */
+ /* Report that we are now swapping relation files XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_SWAP_REL_FILES);
@@ -1417,13 +1423,13 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap,
else if (newrelpersistence == RELPERSISTENCE_PERMANENT)
reindex_flags |= REINDEX_REL_FORCE_INDEXES_PERMANENT;
- /* Report that we are now reindexing relations */
+ /* Report that we are now reindexing relations XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_REBUILD_INDEX);
reindex_relation(OIDOldHeap, reindex_flags, &reindex_params);
- /* Report that we are now doing clean up */
+ /* Report that we are now doing clean up XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_FINAL_CLEANUP);
@@ -1633,9 +1639,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
/* Cluster each relation in a separate transaction */
static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
{
ListCell *lc;
+ off_t childnum = 0;
/* Commit to get out of starting transaction */
PopActiveSnapshot();
@@ -1658,6 +1665,11 @@ cluster_multiple_rels(List *rvs, int options)
cluster_rel(rvtc->tableOid, rvtc->indexOid,
&cluster_params);
+ if (ispartitioned)
+ pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+ childnum++);
+
+
PopActiveSnapshot();
CommitTransactionCommand();
}
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
#define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS 5
#define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED 6
#define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT 7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL 17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE 18
/* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
#define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP 1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b632d9f2ea..d1fea532bc 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1941,7 +1941,9 @@ pg_stat_progress_cluster| SELECT s.pid,
s.param5 AS heap_tuples_written,
s.param6 AS heap_blks_total,
s.param7 AS heap_blks_scanned,
- s.param8 AS index_rebuild_count
+ s.param8 AS index_rebuild_count,
+ s.param18 AS partitions_total,
+ s.param19 AS partitions_done
FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
LEFT JOIN pg_database d ON ((s.datid = d.oid)));
pg_stat_progress_copy| SELECT s.pid,
--
2.17.0
--O5XBE6gyVG5Rl6Rj
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v8-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"
^ permalink raw reply [nested|flat] 73+ messages in thread
* [PATCH v9 3/8] f! progress reporting..
@ 2021-02-09 02:45 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 73+ messages in thread
From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)
This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.
This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
doc/src/sgml/monitoring.sgml | 21 +++++++++++
src/backend/catalog/system_views.sql | 4 +-
src/backend/commands/cluster.c | 55 ++++++++++++++++------------
src/backend/commands/vacuum.c | 5 +++
src/include/commands/progress.h | 3 ++
src/test/regress/expected/rules.out | 4 +-
6 files changed, 67 insertions(+), 25 deletions(-)
diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml
index 3513e127b7..dca47f6a48 100644
--- a/doc/src/sgml/monitoring.sgml
+++ b/doc/src/sgml/monitoring.sgml
@@ -6231,6 +6231,27 @@ SELECT pg_stat_get_backend_pid(s.backendid) AS pid,
is <literal>rebuilding index</literal>.
</para></entry>
</row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>total_partitions</structfield> <type>bigint</type>
+ </para>
+ When clustering a partitioned table, this column is set to the total
+ number of partitions to be clustered.
+ <para>
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>done_partitions</structfield> <type>bigint</type>
+ </para>
+ <para>
+ When clustering a partitioned table, this column is set to the number
+ of partitions which have been clustered.
+ </para></entry>
+ </row>
+
</tbody>
</tgroup>
</table>
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fc94a73a54..21c27f0437 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1070,7 +1070,9 @@ CREATE VIEW pg_stat_progress_cluster AS
S.param5 AS heap_tuples_written,
S.param6 AS heap_blks_total,
S.param7 AS heap_blks_scanned,
- S.param8 AS index_rebuild_count
+ S.param8 AS index_rebuild_count,
+ S.param18 AS partitions_total,
+ S.param19 AS partitions_done
FROM pg_stat_get_progress_info('CLUSTER') AS S
LEFT JOIN pg_database D ON S.datid = D.oid;
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 0c08ac56dc..7d8d1d8a69 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
static List *get_tables_to_cluster(MemoryContext cluster_context);
static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
/*---------------------------------------------------------------------------
@@ -188,16 +188,26 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
/* close relation, keep lock till commit */
table_close(rel, ShareUpdateExclusiveLock);
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
{
/* Do the job. */
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_CLUSTER);
cluster_rel(tableOid, indexOid, ¶ms);
+ pgstat_progress_end_command();
}
else
{
List *rvs;
MemoryContext cluster_context;
+ int64 progress_val[2];
+ const int progress_index[2] = {
+ PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+ };
+
/* Refuse to hold strong locks in a user transaction */
PreventInTransactionBlock(isTopLevel, "CLUSTER");
@@ -206,7 +216,15 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
ALLOCSET_DEFAULT_SIZES);
rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
- cluster_multiple_rels(rvs, params.options);
+
+ /* Report command and number of partitions */
+ progress_val[0] = PROGRESS_CLUSTER_COMMAND_CLUSTER;
+ /* XXX: rvs includes the parent, which is not a "partition" */
+ progress_val[1] = list_length(rvs);
+
+ pgstat_progress_update_multi_param(2, progress_index, progress_val);
+ cluster_multiple_rels(rvs, params.options, true);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -245,7 +263,11 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
* cluster_context.
*/
rvs = get_tables_to_cluster(cluster_context);
- cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_CLUSTER);
+ cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -282,14 +304,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* Check for user-requested abort. */
CHECK_FOR_INTERRUPTS();
- pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
- if (OidIsValid(indexOid))
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_CLUSTER);
- else
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
/*
* We grab exclusive access to the target rel and index for the duration
* of the transaction. (This is redundant for the single-transaction
@@ -300,10 +314,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* If the table has gone away, we can skip processing it */
if (!OldHeap)
- {
- pgstat_progress_end_command();
return;
- }
/*
* Since we may open a new transaction for each relation, we have to check
@@ -319,7 +330,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!pg_class_ownercheck(tableOid, GetUserId()))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -334,7 +344,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (RELATION_IS_OTHER_TEMP(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -346,7 +355,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -357,7 +365,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!get_index_isclustered(indexOid))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
}
@@ -416,7 +423,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!RelationIsPopulated(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -424,7 +430,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -440,8 +445,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
rebuild_relation(OldHeap, indexOid, verbose);
/* NB: rebuild_relation does table_close() on OldHeap */
-
- pgstat_progress_end_command();
}
/*
@@ -1632,9 +1635,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
/* Cluster each relation in a separate transaction */
static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
{
ListCell *lc;
+ off_t childnum = 0;
/* Commit to get out of starting transaction */
PopActiveSnapshot();
@@ -1657,6 +1661,11 @@ cluster_multiple_rels(List *rvs, int options)
cluster_rel(rvtc->tableOid, rvtc->indexOid,
&cluster_params);
+ if (ispartitioned)
+ pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+ ++childnum);
+
+
PopActiveSnapshot();
CommitTransactionCommand();
}
diff --git a/src/backend/commands/vacuum.c b/src/backend/commands/vacuum.c
index c064352e23..05d52858b1 100644
--- a/src/backend/commands/vacuum.c
+++ b/src/backend/commands/vacuum.c
@@ -37,6 +37,7 @@
#include "catalog/pg_namespace.h"
#include "commands/cluster.h"
#include "commands/defrem.h"
+#include "commands/progress.h"
#include "commands/vacuum.h"
#include "miscadmin.h"
#include "nodes/makefuncs.h"
@@ -1937,7 +1938,11 @@ vacuum_rel(Oid relid, RangeVar *relation, VacuumParams *params)
cluster_params.options |= CLUOPT_VERBOSE;
/* VACUUM FULL is now a variant of CLUSTER; see cluster.c */
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, relid);
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
cluster_rel(relid, InvalidOid, &cluster_params);
+ pgstat_progress_end_command();
}
else
table_relation_vacuum(onerel, params, vac_strategy);
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
#define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS 5
#define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED 6
#define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT 7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL 17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE 18
/* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
#define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP 1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b1c9b7bdfe..c888415235 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1943,7 +1943,9 @@ pg_stat_progress_cluster| SELECT s.pid,
s.param5 AS heap_tuples_written,
s.param6 AS heap_blks_total,
s.param7 AS heap_blks_scanned,
- s.param8 AS index_rebuild_count
+ s.param8 AS index_rebuild_count,
+ s.param18 AS partitions_total,
+ s.param19 AS partitions_done
FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
LEFT JOIN pg_database d ON ((s.datid = d.oid)));
pg_stat_progress_copy| SELECT s.pid,
--
2.17.0
--fmvA4kSBHQVZhkR6
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v9-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"
^ permalink raw reply [nested|flat] 73+ messages in thread
* [PATCH v8 3/8] f! progress reporting..
@ 2021-02-09 02:45 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 73+ messages in thread
From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)
This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.
This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
src/backend/catalog/system_views.sql | 4 +-
src/backend/commands/cluster.c | 64 +++++++++++++++++-----------
src/include/commands/progress.h | 3 ++
src/test/regress/expected/rules.out | 4 +-
4 files changed, 47 insertions(+), 28 deletions(-)
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fa58afd9d7..9a3cb5f935 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1069,7 +1069,9 @@ CREATE VIEW pg_stat_progress_cluster AS
S.param5 AS heap_tuples_written,
S.param6 AS heap_blks_total,
S.param7 AS heap_blks_scanned,
- S.param8 AS index_rebuild_count
+ S.param8 AS index_rebuild_count,
+ S.param18 AS partitions_total,
+ S.param19 AS partitions_done
FROM pg_stat_get_progress_info('CLUSTER') AS S
LEFT JOIN pg_database D ON S.datid = D.oid;
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 9b6673867c..cb4fc350c6 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
static List *get_tables_to_cluster(MemoryContext cluster_context);
static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
/*---------------------------------------------------------------------------
@@ -188,16 +188,28 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
/* close relation, keep lock till commit */
table_close(rel, ShareUpdateExclusiveLock);
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
{
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ OidIsValid(indexOid) ? PROGRESS_CLUSTER_COMMAND_CLUSTER :
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
+
/* Do the job. */
cluster_rel(tableOid, indexOid, ¶ms);
+ pgstat_progress_end_command();
}
else
{
List *rvs;
MemoryContext cluster_context;
+ int64 progress_val[2];
+ const int progress_index[] = {
+ PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+ };
+
/* Refuse to hold strong locks in a user transaction */
PreventInTransactionBlock(isTopLevel, "CLUSTER");
@@ -206,7 +218,20 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
ALLOCSET_DEFAULT_SIZES);
rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
- cluster_multiple_rels(rvs, params.options);
+
+ /* Report vacuum full or cluster and number of partitions */
+ progress_val[0] = OidIsValid(indexOid) ?
+ PROGRESS_CLUSTER_COMMAND_CLUSTER :
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL;
+ progress_val[1] = list_length(rvs);
+// This is currently showing the total number of *tables*, including the
+// parent, and intermediate partitions, so the column should be renamed, or
+// we should count the number of leaf partitions
+
+ pgstat_progress_update_multi_param(2, progress_index, progress_val);
+
+ cluster_multiple_rels(rvs, params.options, true);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -245,7 +270,7 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
* cluster_context.
*/
rvs = get_tables_to_cluster(cluster_context);
- cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+ cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -282,14 +307,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* Check for user-requested abort. */
CHECK_FOR_INTERRUPTS();
- pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
- if (OidIsValid(indexOid))
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_CLUSTER);
- else
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
/*
* We grab exclusive access to the target rel and index for the duration
* of the transaction. (This is redundant for the single-transaction
@@ -300,10 +317,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* If the table has gone away, we can skip processing it */
if (!OldHeap)
- {
- pgstat_progress_end_command();
return;
- }
/*
* Since we may open a new transaction for each relation, we have to check
@@ -319,7 +333,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!pg_class_ownercheck(tableOid, GetUserId()))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -334,7 +347,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (RELATION_IS_OTHER_TEMP(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -346,7 +358,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -357,7 +368,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!get_index_isclustered(indexOid))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
}
@@ -416,7 +426,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!RelationIsPopulated(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -424,7 +433,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -440,8 +448,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
rebuild_relation(OldHeap, indexOid, verbose);
/* NB: rebuild_relation does table_close() on OldHeap */
-
- pgstat_progress_end_command();
}
/*
@@ -1366,7 +1372,7 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap,
ReindexParams reindex_params = {0};
int i;
- /* Report that we are now swapping relation files */
+ /* Report that we are now swapping relation files XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_SWAP_REL_FILES);
@@ -1417,13 +1423,13 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap,
else if (newrelpersistence == RELPERSISTENCE_PERMANENT)
reindex_flags |= REINDEX_REL_FORCE_INDEXES_PERMANENT;
- /* Report that we are now reindexing relations */
+ /* Report that we are now reindexing relations XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_REBUILD_INDEX);
reindex_relation(OIDOldHeap, reindex_flags, &reindex_params);
- /* Report that we are now doing clean up */
+ /* Report that we are now doing clean up XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_FINAL_CLEANUP);
@@ -1633,9 +1639,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
/* Cluster each relation in a separate transaction */
static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
{
ListCell *lc;
+ off_t childnum = 0;
/* Commit to get out of starting transaction */
PopActiveSnapshot();
@@ -1658,6 +1665,11 @@ cluster_multiple_rels(List *rvs, int options)
cluster_rel(rvtc->tableOid, rvtc->indexOid,
&cluster_params);
+ if (ispartitioned)
+ pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+ childnum++);
+
+
PopActiveSnapshot();
CommitTransactionCommand();
}
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
#define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS 5
#define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED 6
#define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT 7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL 17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE 18
/* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
#define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP 1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b632d9f2ea..d1fea532bc 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1941,7 +1941,9 @@ pg_stat_progress_cluster| SELECT s.pid,
s.param5 AS heap_tuples_written,
s.param6 AS heap_blks_total,
s.param7 AS heap_blks_scanned,
- s.param8 AS index_rebuild_count
+ s.param8 AS index_rebuild_count,
+ s.param18 AS partitions_total,
+ s.param19 AS partitions_done
FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
LEFT JOIN pg_database d ON ((s.datid = d.oid)));
pg_stat_progress_copy| SELECT s.pid,
--
2.17.0
--O5XBE6gyVG5Rl6Rj
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v8-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"
^ permalink raw reply [nested|flat] 73+ messages in thread
* [PATCH v8 3/8] f! progress reporting..
@ 2021-02-09 02:45 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 73+ messages in thread
From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)
This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.
This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
src/backend/catalog/system_views.sql | 4 +-
src/backend/commands/cluster.c | 64 +++++++++++++++++-----------
src/include/commands/progress.h | 3 ++
src/test/regress/expected/rules.out | 4 +-
4 files changed, 47 insertions(+), 28 deletions(-)
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fa58afd9d7..9a3cb5f935 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1069,7 +1069,9 @@ CREATE VIEW pg_stat_progress_cluster AS
S.param5 AS heap_tuples_written,
S.param6 AS heap_blks_total,
S.param7 AS heap_blks_scanned,
- S.param8 AS index_rebuild_count
+ S.param8 AS index_rebuild_count,
+ S.param18 AS partitions_total,
+ S.param19 AS partitions_done
FROM pg_stat_get_progress_info('CLUSTER') AS S
LEFT JOIN pg_database D ON S.datid = D.oid;
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 9b6673867c..cb4fc350c6 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
static List *get_tables_to_cluster(MemoryContext cluster_context);
static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
/*---------------------------------------------------------------------------
@@ -188,16 +188,28 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
/* close relation, keep lock till commit */
table_close(rel, ShareUpdateExclusiveLock);
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
{
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ OidIsValid(indexOid) ? PROGRESS_CLUSTER_COMMAND_CLUSTER :
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
+
/* Do the job. */
cluster_rel(tableOid, indexOid, ¶ms);
+ pgstat_progress_end_command();
}
else
{
List *rvs;
MemoryContext cluster_context;
+ int64 progress_val[2];
+ const int progress_index[] = {
+ PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+ };
+
/* Refuse to hold strong locks in a user transaction */
PreventInTransactionBlock(isTopLevel, "CLUSTER");
@@ -206,7 +218,20 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
ALLOCSET_DEFAULT_SIZES);
rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
- cluster_multiple_rels(rvs, params.options);
+
+ /* Report vacuum full or cluster and number of partitions */
+ progress_val[0] = OidIsValid(indexOid) ?
+ PROGRESS_CLUSTER_COMMAND_CLUSTER :
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL;
+ progress_val[1] = list_length(rvs);
+// This is currently showing the total number of *tables*, including the
+// parent, and intermediate partitions, so the column should be renamed, or
+// we should count the number of leaf partitions
+
+ pgstat_progress_update_multi_param(2, progress_index, progress_val);
+
+ cluster_multiple_rels(rvs, params.options, true);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -245,7 +270,7 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
* cluster_context.
*/
rvs = get_tables_to_cluster(cluster_context);
- cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+ cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -282,14 +307,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* Check for user-requested abort. */
CHECK_FOR_INTERRUPTS();
- pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
- if (OidIsValid(indexOid))
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_CLUSTER);
- else
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
/*
* We grab exclusive access to the target rel and index for the duration
* of the transaction. (This is redundant for the single-transaction
@@ -300,10 +317,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* If the table has gone away, we can skip processing it */
if (!OldHeap)
- {
- pgstat_progress_end_command();
return;
- }
/*
* Since we may open a new transaction for each relation, we have to check
@@ -319,7 +333,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!pg_class_ownercheck(tableOid, GetUserId()))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -334,7 +347,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (RELATION_IS_OTHER_TEMP(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -346,7 +358,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -357,7 +368,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!get_index_isclustered(indexOid))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
}
@@ -416,7 +426,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!RelationIsPopulated(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -424,7 +433,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -440,8 +448,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
rebuild_relation(OldHeap, indexOid, verbose);
/* NB: rebuild_relation does table_close() on OldHeap */
-
- pgstat_progress_end_command();
}
/*
@@ -1366,7 +1372,7 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap,
ReindexParams reindex_params = {0};
int i;
- /* Report that we are now swapping relation files */
+ /* Report that we are now swapping relation files XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_SWAP_REL_FILES);
@@ -1417,13 +1423,13 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap,
else if (newrelpersistence == RELPERSISTENCE_PERMANENT)
reindex_flags |= REINDEX_REL_FORCE_INDEXES_PERMANENT;
- /* Report that we are now reindexing relations */
+ /* Report that we are now reindexing relations XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_REBUILD_INDEX);
reindex_relation(OIDOldHeap, reindex_flags, &reindex_params);
- /* Report that we are now doing clean up */
+ /* Report that we are now doing clean up XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_FINAL_CLEANUP);
@@ -1633,9 +1639,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
/* Cluster each relation in a separate transaction */
static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
{
ListCell *lc;
+ off_t childnum = 0;
/* Commit to get out of starting transaction */
PopActiveSnapshot();
@@ -1658,6 +1665,11 @@ cluster_multiple_rels(List *rvs, int options)
cluster_rel(rvtc->tableOid, rvtc->indexOid,
&cluster_params);
+ if (ispartitioned)
+ pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+ childnum++);
+
+
PopActiveSnapshot();
CommitTransactionCommand();
}
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
#define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS 5
#define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED 6
#define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT 7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL 17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE 18
/* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
#define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP 1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b632d9f2ea..d1fea532bc 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1941,7 +1941,9 @@ pg_stat_progress_cluster| SELECT s.pid,
s.param5 AS heap_tuples_written,
s.param6 AS heap_blks_total,
s.param7 AS heap_blks_scanned,
- s.param8 AS index_rebuild_count
+ s.param8 AS index_rebuild_count,
+ s.param18 AS partitions_total,
+ s.param19 AS partitions_done
FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
LEFT JOIN pg_database d ON ((s.datid = d.oid)));
pg_stat_progress_copy| SELECT s.pid,
--
2.17.0
--O5XBE6gyVG5Rl6Rj
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v8-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"
^ permalink raw reply [nested|flat] 73+ messages in thread
* [PATCH v8 3/8] f! progress reporting..
@ 2021-02-09 02:45 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 73+ messages in thread
From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)
This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.
This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
src/backend/catalog/system_views.sql | 4 +-
src/backend/commands/cluster.c | 64 +++++++++++++++++-----------
src/include/commands/progress.h | 3 ++
src/test/regress/expected/rules.out | 4 +-
4 files changed, 47 insertions(+), 28 deletions(-)
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fa58afd9d7..9a3cb5f935 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1069,7 +1069,9 @@ CREATE VIEW pg_stat_progress_cluster AS
S.param5 AS heap_tuples_written,
S.param6 AS heap_blks_total,
S.param7 AS heap_blks_scanned,
- S.param8 AS index_rebuild_count
+ S.param8 AS index_rebuild_count,
+ S.param18 AS partitions_total,
+ S.param19 AS partitions_done
FROM pg_stat_get_progress_info('CLUSTER') AS S
LEFT JOIN pg_database D ON S.datid = D.oid;
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 9b6673867c..cb4fc350c6 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
static List *get_tables_to_cluster(MemoryContext cluster_context);
static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
/*---------------------------------------------------------------------------
@@ -188,16 +188,28 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
/* close relation, keep lock till commit */
table_close(rel, ShareUpdateExclusiveLock);
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
{
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ OidIsValid(indexOid) ? PROGRESS_CLUSTER_COMMAND_CLUSTER :
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
+
/* Do the job. */
cluster_rel(tableOid, indexOid, ¶ms);
+ pgstat_progress_end_command();
}
else
{
List *rvs;
MemoryContext cluster_context;
+ int64 progress_val[2];
+ const int progress_index[] = {
+ PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+ };
+
/* Refuse to hold strong locks in a user transaction */
PreventInTransactionBlock(isTopLevel, "CLUSTER");
@@ -206,7 +218,20 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
ALLOCSET_DEFAULT_SIZES);
rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
- cluster_multiple_rels(rvs, params.options);
+
+ /* Report vacuum full or cluster and number of partitions */
+ progress_val[0] = OidIsValid(indexOid) ?
+ PROGRESS_CLUSTER_COMMAND_CLUSTER :
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL;
+ progress_val[1] = list_length(rvs);
+// This is currently showing the total number of *tables*, including the
+// parent, and intermediate partitions, so the column should be renamed, or
+// we should count the number of leaf partitions
+
+ pgstat_progress_update_multi_param(2, progress_index, progress_val);
+
+ cluster_multiple_rels(rvs, params.options, true);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -245,7 +270,7 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
* cluster_context.
*/
rvs = get_tables_to_cluster(cluster_context);
- cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+ cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -282,14 +307,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* Check for user-requested abort. */
CHECK_FOR_INTERRUPTS();
- pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
- if (OidIsValid(indexOid))
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_CLUSTER);
- else
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
/*
* We grab exclusive access to the target rel and index for the duration
* of the transaction. (This is redundant for the single-transaction
@@ -300,10 +317,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* If the table has gone away, we can skip processing it */
if (!OldHeap)
- {
- pgstat_progress_end_command();
return;
- }
/*
* Since we may open a new transaction for each relation, we have to check
@@ -319,7 +333,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!pg_class_ownercheck(tableOid, GetUserId()))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -334,7 +347,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (RELATION_IS_OTHER_TEMP(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -346,7 +358,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -357,7 +368,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!get_index_isclustered(indexOid))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
}
@@ -416,7 +426,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!RelationIsPopulated(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -424,7 +433,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -440,8 +448,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
rebuild_relation(OldHeap, indexOid, verbose);
/* NB: rebuild_relation does table_close() on OldHeap */
-
- pgstat_progress_end_command();
}
/*
@@ -1366,7 +1372,7 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap,
ReindexParams reindex_params = {0};
int i;
- /* Report that we are now swapping relation files */
+ /* Report that we are now swapping relation files XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_SWAP_REL_FILES);
@@ -1417,13 +1423,13 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap,
else if (newrelpersistence == RELPERSISTENCE_PERMANENT)
reindex_flags |= REINDEX_REL_FORCE_INDEXES_PERMANENT;
- /* Report that we are now reindexing relations */
+ /* Report that we are now reindexing relations XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_REBUILD_INDEX);
reindex_relation(OIDOldHeap, reindex_flags, &reindex_params);
- /* Report that we are now doing clean up */
+ /* Report that we are now doing clean up XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_FINAL_CLEANUP);
@@ -1633,9 +1639,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
/* Cluster each relation in a separate transaction */
static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
{
ListCell *lc;
+ off_t childnum = 0;
/* Commit to get out of starting transaction */
PopActiveSnapshot();
@@ -1658,6 +1665,11 @@ cluster_multiple_rels(List *rvs, int options)
cluster_rel(rvtc->tableOid, rvtc->indexOid,
&cluster_params);
+ if (ispartitioned)
+ pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+ childnum++);
+
+
PopActiveSnapshot();
CommitTransactionCommand();
}
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
#define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS 5
#define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED 6
#define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT 7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL 17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE 18
/* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
#define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP 1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b632d9f2ea..d1fea532bc 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1941,7 +1941,9 @@ pg_stat_progress_cluster| SELECT s.pid,
s.param5 AS heap_tuples_written,
s.param6 AS heap_blks_total,
s.param7 AS heap_blks_scanned,
- s.param8 AS index_rebuild_count
+ s.param8 AS index_rebuild_count,
+ s.param18 AS partitions_total,
+ s.param19 AS partitions_done
FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
LEFT JOIN pg_database d ON ((s.datid = d.oid)));
pg_stat_progress_copy| SELECT s.pid,
--
2.17.0
--O5XBE6gyVG5Rl6Rj
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v8-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"
^ permalink raw reply [nested|flat] 73+ messages in thread
* [PATCH v9 3/8] f! progress reporting..
@ 2021-02-09 02:45 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 73+ messages in thread
From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)
This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.
This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
doc/src/sgml/monitoring.sgml | 21 +++++++++++
src/backend/catalog/system_views.sql | 4 +-
src/backend/commands/cluster.c | 55 ++++++++++++++++------------
src/backend/commands/vacuum.c | 5 +++
src/include/commands/progress.h | 3 ++
src/test/regress/expected/rules.out | 4 +-
6 files changed, 67 insertions(+), 25 deletions(-)
diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml
index 3513e127b7..dca47f6a48 100644
--- a/doc/src/sgml/monitoring.sgml
+++ b/doc/src/sgml/monitoring.sgml
@@ -6231,6 +6231,27 @@ SELECT pg_stat_get_backend_pid(s.backendid) AS pid,
is <literal>rebuilding index</literal>.
</para></entry>
</row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>total_partitions</structfield> <type>bigint</type>
+ </para>
+ When clustering a partitioned table, this column is set to the total
+ number of partitions to be clustered.
+ <para>
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>done_partitions</structfield> <type>bigint</type>
+ </para>
+ <para>
+ When clustering a partitioned table, this column is set to the number
+ of partitions which have been clustered.
+ </para></entry>
+ </row>
+
</tbody>
</tgroup>
</table>
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fc94a73a54..21c27f0437 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1070,7 +1070,9 @@ CREATE VIEW pg_stat_progress_cluster AS
S.param5 AS heap_tuples_written,
S.param6 AS heap_blks_total,
S.param7 AS heap_blks_scanned,
- S.param8 AS index_rebuild_count
+ S.param8 AS index_rebuild_count,
+ S.param18 AS partitions_total,
+ S.param19 AS partitions_done
FROM pg_stat_get_progress_info('CLUSTER') AS S
LEFT JOIN pg_database D ON S.datid = D.oid;
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 0c08ac56dc..7d8d1d8a69 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
static List *get_tables_to_cluster(MemoryContext cluster_context);
static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
/*---------------------------------------------------------------------------
@@ -188,16 +188,26 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
/* close relation, keep lock till commit */
table_close(rel, ShareUpdateExclusiveLock);
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
{
/* Do the job. */
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_CLUSTER);
cluster_rel(tableOid, indexOid, ¶ms);
+ pgstat_progress_end_command();
}
else
{
List *rvs;
MemoryContext cluster_context;
+ int64 progress_val[2];
+ const int progress_index[2] = {
+ PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+ };
+
/* Refuse to hold strong locks in a user transaction */
PreventInTransactionBlock(isTopLevel, "CLUSTER");
@@ -206,7 +216,15 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
ALLOCSET_DEFAULT_SIZES);
rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
- cluster_multiple_rels(rvs, params.options);
+
+ /* Report command and number of partitions */
+ progress_val[0] = PROGRESS_CLUSTER_COMMAND_CLUSTER;
+ /* XXX: rvs includes the parent, which is not a "partition" */
+ progress_val[1] = list_length(rvs);
+
+ pgstat_progress_update_multi_param(2, progress_index, progress_val);
+ cluster_multiple_rels(rvs, params.options, true);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -245,7 +263,11 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
* cluster_context.
*/
rvs = get_tables_to_cluster(cluster_context);
- cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_CLUSTER);
+ cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -282,14 +304,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* Check for user-requested abort. */
CHECK_FOR_INTERRUPTS();
- pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
- if (OidIsValid(indexOid))
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_CLUSTER);
- else
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
/*
* We grab exclusive access to the target rel and index for the duration
* of the transaction. (This is redundant for the single-transaction
@@ -300,10 +314,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* If the table has gone away, we can skip processing it */
if (!OldHeap)
- {
- pgstat_progress_end_command();
return;
- }
/*
* Since we may open a new transaction for each relation, we have to check
@@ -319,7 +330,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!pg_class_ownercheck(tableOid, GetUserId()))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -334,7 +344,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (RELATION_IS_OTHER_TEMP(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -346,7 +355,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -357,7 +365,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!get_index_isclustered(indexOid))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
}
@@ -416,7 +423,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!RelationIsPopulated(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -424,7 +430,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -440,8 +445,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
rebuild_relation(OldHeap, indexOid, verbose);
/* NB: rebuild_relation does table_close() on OldHeap */
-
- pgstat_progress_end_command();
}
/*
@@ -1632,9 +1635,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
/* Cluster each relation in a separate transaction */
static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
{
ListCell *lc;
+ off_t childnum = 0;
/* Commit to get out of starting transaction */
PopActiveSnapshot();
@@ -1657,6 +1661,11 @@ cluster_multiple_rels(List *rvs, int options)
cluster_rel(rvtc->tableOid, rvtc->indexOid,
&cluster_params);
+ if (ispartitioned)
+ pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+ ++childnum);
+
+
PopActiveSnapshot();
CommitTransactionCommand();
}
diff --git a/src/backend/commands/vacuum.c b/src/backend/commands/vacuum.c
index c064352e23..05d52858b1 100644
--- a/src/backend/commands/vacuum.c
+++ b/src/backend/commands/vacuum.c
@@ -37,6 +37,7 @@
#include "catalog/pg_namespace.h"
#include "commands/cluster.h"
#include "commands/defrem.h"
+#include "commands/progress.h"
#include "commands/vacuum.h"
#include "miscadmin.h"
#include "nodes/makefuncs.h"
@@ -1937,7 +1938,11 @@ vacuum_rel(Oid relid, RangeVar *relation, VacuumParams *params)
cluster_params.options |= CLUOPT_VERBOSE;
/* VACUUM FULL is now a variant of CLUSTER; see cluster.c */
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, relid);
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
cluster_rel(relid, InvalidOid, &cluster_params);
+ pgstat_progress_end_command();
}
else
table_relation_vacuum(onerel, params, vac_strategy);
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
#define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS 5
#define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED 6
#define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT 7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL 17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE 18
/* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
#define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP 1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b1c9b7bdfe..c888415235 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1943,7 +1943,9 @@ pg_stat_progress_cluster| SELECT s.pid,
s.param5 AS heap_tuples_written,
s.param6 AS heap_blks_total,
s.param7 AS heap_blks_scanned,
- s.param8 AS index_rebuild_count
+ s.param8 AS index_rebuild_count,
+ s.param18 AS partitions_total,
+ s.param19 AS partitions_done
FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
LEFT JOIN pg_database d ON ((s.datid = d.oid)));
pg_stat_progress_copy| SELECT s.pid,
--
2.17.0
--fmvA4kSBHQVZhkR6
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v9-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"
^ permalink raw reply [nested|flat] 73+ messages in thread
* [PATCH v9 3/8] f! progress reporting..
@ 2021-02-09 02:45 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 73+ messages in thread
From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)
This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.
This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
doc/src/sgml/monitoring.sgml | 21 +++++++++++
src/backend/catalog/system_views.sql | 4 +-
src/backend/commands/cluster.c | 55 ++++++++++++++++------------
src/backend/commands/vacuum.c | 5 +++
src/include/commands/progress.h | 3 ++
src/test/regress/expected/rules.out | 4 +-
6 files changed, 67 insertions(+), 25 deletions(-)
diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml
index 3513e127b7..dca47f6a48 100644
--- a/doc/src/sgml/monitoring.sgml
+++ b/doc/src/sgml/monitoring.sgml
@@ -6231,6 +6231,27 @@ SELECT pg_stat_get_backend_pid(s.backendid) AS pid,
is <literal>rebuilding index</literal>.
</para></entry>
</row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>total_partitions</structfield> <type>bigint</type>
+ </para>
+ When clustering a partitioned table, this column is set to the total
+ number of partitions to be clustered.
+ <para>
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>done_partitions</structfield> <type>bigint</type>
+ </para>
+ <para>
+ When clustering a partitioned table, this column is set to the number
+ of partitions which have been clustered.
+ </para></entry>
+ </row>
+
</tbody>
</tgroup>
</table>
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fc94a73a54..21c27f0437 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1070,7 +1070,9 @@ CREATE VIEW pg_stat_progress_cluster AS
S.param5 AS heap_tuples_written,
S.param6 AS heap_blks_total,
S.param7 AS heap_blks_scanned,
- S.param8 AS index_rebuild_count
+ S.param8 AS index_rebuild_count,
+ S.param18 AS partitions_total,
+ S.param19 AS partitions_done
FROM pg_stat_get_progress_info('CLUSTER') AS S
LEFT JOIN pg_database D ON S.datid = D.oid;
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 0c08ac56dc..7d8d1d8a69 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
static List *get_tables_to_cluster(MemoryContext cluster_context);
static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
/*---------------------------------------------------------------------------
@@ -188,16 +188,26 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
/* close relation, keep lock till commit */
table_close(rel, ShareUpdateExclusiveLock);
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
{
/* Do the job. */
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_CLUSTER);
cluster_rel(tableOid, indexOid, ¶ms);
+ pgstat_progress_end_command();
}
else
{
List *rvs;
MemoryContext cluster_context;
+ int64 progress_val[2];
+ const int progress_index[2] = {
+ PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+ };
+
/* Refuse to hold strong locks in a user transaction */
PreventInTransactionBlock(isTopLevel, "CLUSTER");
@@ -206,7 +216,15 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
ALLOCSET_DEFAULT_SIZES);
rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
- cluster_multiple_rels(rvs, params.options);
+
+ /* Report command and number of partitions */
+ progress_val[0] = PROGRESS_CLUSTER_COMMAND_CLUSTER;
+ /* XXX: rvs includes the parent, which is not a "partition" */
+ progress_val[1] = list_length(rvs);
+
+ pgstat_progress_update_multi_param(2, progress_index, progress_val);
+ cluster_multiple_rels(rvs, params.options, true);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -245,7 +263,11 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
* cluster_context.
*/
rvs = get_tables_to_cluster(cluster_context);
- cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_CLUSTER);
+ cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -282,14 +304,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* Check for user-requested abort. */
CHECK_FOR_INTERRUPTS();
- pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
- if (OidIsValid(indexOid))
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_CLUSTER);
- else
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
/*
* We grab exclusive access to the target rel and index for the duration
* of the transaction. (This is redundant for the single-transaction
@@ -300,10 +314,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* If the table has gone away, we can skip processing it */
if (!OldHeap)
- {
- pgstat_progress_end_command();
return;
- }
/*
* Since we may open a new transaction for each relation, we have to check
@@ -319,7 +330,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!pg_class_ownercheck(tableOid, GetUserId()))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -334,7 +344,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (RELATION_IS_OTHER_TEMP(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -346,7 +355,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -357,7 +365,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!get_index_isclustered(indexOid))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
}
@@ -416,7 +423,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!RelationIsPopulated(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -424,7 +430,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -440,8 +445,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
rebuild_relation(OldHeap, indexOid, verbose);
/* NB: rebuild_relation does table_close() on OldHeap */
-
- pgstat_progress_end_command();
}
/*
@@ -1632,9 +1635,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
/* Cluster each relation in a separate transaction */
static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
{
ListCell *lc;
+ off_t childnum = 0;
/* Commit to get out of starting transaction */
PopActiveSnapshot();
@@ -1657,6 +1661,11 @@ cluster_multiple_rels(List *rvs, int options)
cluster_rel(rvtc->tableOid, rvtc->indexOid,
&cluster_params);
+ if (ispartitioned)
+ pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+ ++childnum);
+
+
PopActiveSnapshot();
CommitTransactionCommand();
}
diff --git a/src/backend/commands/vacuum.c b/src/backend/commands/vacuum.c
index c064352e23..05d52858b1 100644
--- a/src/backend/commands/vacuum.c
+++ b/src/backend/commands/vacuum.c
@@ -37,6 +37,7 @@
#include "catalog/pg_namespace.h"
#include "commands/cluster.h"
#include "commands/defrem.h"
+#include "commands/progress.h"
#include "commands/vacuum.h"
#include "miscadmin.h"
#include "nodes/makefuncs.h"
@@ -1937,7 +1938,11 @@ vacuum_rel(Oid relid, RangeVar *relation, VacuumParams *params)
cluster_params.options |= CLUOPT_VERBOSE;
/* VACUUM FULL is now a variant of CLUSTER; see cluster.c */
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, relid);
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
cluster_rel(relid, InvalidOid, &cluster_params);
+ pgstat_progress_end_command();
}
else
table_relation_vacuum(onerel, params, vac_strategy);
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
#define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS 5
#define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED 6
#define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT 7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL 17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE 18
/* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
#define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP 1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b1c9b7bdfe..c888415235 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1943,7 +1943,9 @@ pg_stat_progress_cluster| SELECT s.pid,
s.param5 AS heap_tuples_written,
s.param6 AS heap_blks_total,
s.param7 AS heap_blks_scanned,
- s.param8 AS index_rebuild_count
+ s.param8 AS index_rebuild_count,
+ s.param18 AS partitions_total,
+ s.param19 AS partitions_done
FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
LEFT JOIN pg_database d ON ((s.datid = d.oid)));
pg_stat_progress_copy| SELECT s.pid,
--
2.17.0
--fmvA4kSBHQVZhkR6
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v9-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"
^ permalink raw reply [nested|flat] 73+ messages in thread
* [PATCH v9 3/8] f! progress reporting..
@ 2021-02-09 02:45 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 73+ messages in thread
From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)
This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.
This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
doc/src/sgml/monitoring.sgml | 21 +++++++++++
src/backend/catalog/system_views.sql | 4 +-
src/backend/commands/cluster.c | 55 ++++++++++++++++------------
src/backend/commands/vacuum.c | 5 +++
src/include/commands/progress.h | 3 ++
src/test/regress/expected/rules.out | 4 +-
6 files changed, 67 insertions(+), 25 deletions(-)
diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml
index 3513e127b7..dca47f6a48 100644
--- a/doc/src/sgml/monitoring.sgml
+++ b/doc/src/sgml/monitoring.sgml
@@ -6231,6 +6231,27 @@ SELECT pg_stat_get_backend_pid(s.backendid) AS pid,
is <literal>rebuilding index</literal>.
</para></entry>
</row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>total_partitions</structfield> <type>bigint</type>
+ </para>
+ When clustering a partitioned table, this column is set to the total
+ number of partitions to be clustered.
+ <para>
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>done_partitions</structfield> <type>bigint</type>
+ </para>
+ <para>
+ When clustering a partitioned table, this column is set to the number
+ of partitions which have been clustered.
+ </para></entry>
+ </row>
+
</tbody>
</tgroup>
</table>
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fc94a73a54..21c27f0437 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1070,7 +1070,9 @@ CREATE VIEW pg_stat_progress_cluster AS
S.param5 AS heap_tuples_written,
S.param6 AS heap_blks_total,
S.param7 AS heap_blks_scanned,
- S.param8 AS index_rebuild_count
+ S.param8 AS index_rebuild_count,
+ S.param18 AS partitions_total,
+ S.param19 AS partitions_done
FROM pg_stat_get_progress_info('CLUSTER') AS S
LEFT JOIN pg_database D ON S.datid = D.oid;
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 0c08ac56dc..7d8d1d8a69 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
static List *get_tables_to_cluster(MemoryContext cluster_context);
static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
/*---------------------------------------------------------------------------
@@ -188,16 +188,26 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
/* close relation, keep lock till commit */
table_close(rel, ShareUpdateExclusiveLock);
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
{
/* Do the job. */
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_CLUSTER);
cluster_rel(tableOid, indexOid, ¶ms);
+ pgstat_progress_end_command();
}
else
{
List *rvs;
MemoryContext cluster_context;
+ int64 progress_val[2];
+ const int progress_index[2] = {
+ PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+ };
+
/* Refuse to hold strong locks in a user transaction */
PreventInTransactionBlock(isTopLevel, "CLUSTER");
@@ -206,7 +216,15 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
ALLOCSET_DEFAULT_SIZES);
rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
- cluster_multiple_rels(rvs, params.options);
+
+ /* Report command and number of partitions */
+ progress_val[0] = PROGRESS_CLUSTER_COMMAND_CLUSTER;
+ /* XXX: rvs includes the parent, which is not a "partition" */
+ progress_val[1] = list_length(rvs);
+
+ pgstat_progress_update_multi_param(2, progress_index, progress_val);
+ cluster_multiple_rels(rvs, params.options, true);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -245,7 +263,11 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
* cluster_context.
*/
rvs = get_tables_to_cluster(cluster_context);
- cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_CLUSTER);
+ cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -282,14 +304,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* Check for user-requested abort. */
CHECK_FOR_INTERRUPTS();
- pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
- if (OidIsValid(indexOid))
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_CLUSTER);
- else
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
/*
* We grab exclusive access to the target rel and index for the duration
* of the transaction. (This is redundant for the single-transaction
@@ -300,10 +314,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* If the table has gone away, we can skip processing it */
if (!OldHeap)
- {
- pgstat_progress_end_command();
return;
- }
/*
* Since we may open a new transaction for each relation, we have to check
@@ -319,7 +330,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!pg_class_ownercheck(tableOid, GetUserId()))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -334,7 +344,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (RELATION_IS_OTHER_TEMP(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -346,7 +355,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -357,7 +365,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!get_index_isclustered(indexOid))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
}
@@ -416,7 +423,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!RelationIsPopulated(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -424,7 +430,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -440,8 +445,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
rebuild_relation(OldHeap, indexOid, verbose);
/* NB: rebuild_relation does table_close() on OldHeap */
-
- pgstat_progress_end_command();
}
/*
@@ -1632,9 +1635,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
/* Cluster each relation in a separate transaction */
static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
{
ListCell *lc;
+ off_t childnum = 0;
/* Commit to get out of starting transaction */
PopActiveSnapshot();
@@ -1657,6 +1661,11 @@ cluster_multiple_rels(List *rvs, int options)
cluster_rel(rvtc->tableOid, rvtc->indexOid,
&cluster_params);
+ if (ispartitioned)
+ pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+ ++childnum);
+
+
PopActiveSnapshot();
CommitTransactionCommand();
}
diff --git a/src/backend/commands/vacuum.c b/src/backend/commands/vacuum.c
index c064352e23..05d52858b1 100644
--- a/src/backend/commands/vacuum.c
+++ b/src/backend/commands/vacuum.c
@@ -37,6 +37,7 @@
#include "catalog/pg_namespace.h"
#include "commands/cluster.h"
#include "commands/defrem.h"
+#include "commands/progress.h"
#include "commands/vacuum.h"
#include "miscadmin.h"
#include "nodes/makefuncs.h"
@@ -1937,7 +1938,11 @@ vacuum_rel(Oid relid, RangeVar *relation, VacuumParams *params)
cluster_params.options |= CLUOPT_VERBOSE;
/* VACUUM FULL is now a variant of CLUSTER; see cluster.c */
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, relid);
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
cluster_rel(relid, InvalidOid, &cluster_params);
+ pgstat_progress_end_command();
}
else
table_relation_vacuum(onerel, params, vac_strategy);
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
#define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS 5
#define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED 6
#define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT 7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL 17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE 18
/* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
#define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP 1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b1c9b7bdfe..c888415235 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1943,7 +1943,9 @@ pg_stat_progress_cluster| SELECT s.pid,
s.param5 AS heap_tuples_written,
s.param6 AS heap_blks_total,
s.param7 AS heap_blks_scanned,
- s.param8 AS index_rebuild_count
+ s.param8 AS index_rebuild_count,
+ s.param18 AS partitions_total,
+ s.param19 AS partitions_done
FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
LEFT JOIN pg_database d ON ((s.datid = d.oid)));
pg_stat_progress_copy| SELECT s.pid,
--
2.17.0
--fmvA4kSBHQVZhkR6
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v9-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"
^ permalink raw reply [nested|flat] 73+ messages in thread
* [PATCH v9 3/8] f! progress reporting..
@ 2021-02-09 02:45 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 73+ messages in thread
From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)
This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.
This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
doc/src/sgml/monitoring.sgml | 21 +++++++++++
src/backend/catalog/system_views.sql | 4 +-
src/backend/commands/cluster.c | 55 ++++++++++++++++------------
src/backend/commands/vacuum.c | 5 +++
src/include/commands/progress.h | 3 ++
src/test/regress/expected/rules.out | 4 +-
6 files changed, 67 insertions(+), 25 deletions(-)
diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml
index 3513e127b7..dca47f6a48 100644
--- a/doc/src/sgml/monitoring.sgml
+++ b/doc/src/sgml/monitoring.sgml
@@ -6231,6 +6231,27 @@ SELECT pg_stat_get_backend_pid(s.backendid) AS pid,
is <literal>rebuilding index</literal>.
</para></entry>
</row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>total_partitions</structfield> <type>bigint</type>
+ </para>
+ When clustering a partitioned table, this column is set to the total
+ number of partitions to be clustered.
+ <para>
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>done_partitions</structfield> <type>bigint</type>
+ </para>
+ <para>
+ When clustering a partitioned table, this column is set to the number
+ of partitions which have been clustered.
+ </para></entry>
+ </row>
+
</tbody>
</tgroup>
</table>
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fc94a73a54..21c27f0437 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1070,7 +1070,9 @@ CREATE VIEW pg_stat_progress_cluster AS
S.param5 AS heap_tuples_written,
S.param6 AS heap_blks_total,
S.param7 AS heap_blks_scanned,
- S.param8 AS index_rebuild_count
+ S.param8 AS index_rebuild_count,
+ S.param18 AS partitions_total,
+ S.param19 AS partitions_done
FROM pg_stat_get_progress_info('CLUSTER') AS S
LEFT JOIN pg_database D ON S.datid = D.oid;
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 0c08ac56dc..7d8d1d8a69 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
static List *get_tables_to_cluster(MemoryContext cluster_context);
static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
/*---------------------------------------------------------------------------
@@ -188,16 +188,26 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
/* close relation, keep lock till commit */
table_close(rel, ShareUpdateExclusiveLock);
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
{
/* Do the job. */
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_CLUSTER);
cluster_rel(tableOid, indexOid, ¶ms);
+ pgstat_progress_end_command();
}
else
{
List *rvs;
MemoryContext cluster_context;
+ int64 progress_val[2];
+ const int progress_index[2] = {
+ PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+ };
+
/* Refuse to hold strong locks in a user transaction */
PreventInTransactionBlock(isTopLevel, "CLUSTER");
@@ -206,7 +216,15 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
ALLOCSET_DEFAULT_SIZES);
rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
- cluster_multiple_rels(rvs, params.options);
+
+ /* Report command and number of partitions */
+ progress_val[0] = PROGRESS_CLUSTER_COMMAND_CLUSTER;
+ /* XXX: rvs includes the parent, which is not a "partition" */
+ progress_val[1] = list_length(rvs);
+
+ pgstat_progress_update_multi_param(2, progress_index, progress_val);
+ cluster_multiple_rels(rvs, params.options, true);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -245,7 +263,11 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
* cluster_context.
*/
rvs = get_tables_to_cluster(cluster_context);
- cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_CLUSTER);
+ cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -282,14 +304,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* Check for user-requested abort. */
CHECK_FOR_INTERRUPTS();
- pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
- if (OidIsValid(indexOid))
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_CLUSTER);
- else
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
/*
* We grab exclusive access to the target rel and index for the duration
* of the transaction. (This is redundant for the single-transaction
@@ -300,10 +314,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* If the table has gone away, we can skip processing it */
if (!OldHeap)
- {
- pgstat_progress_end_command();
return;
- }
/*
* Since we may open a new transaction for each relation, we have to check
@@ -319,7 +330,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!pg_class_ownercheck(tableOid, GetUserId()))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -334,7 +344,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (RELATION_IS_OTHER_TEMP(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -346,7 +355,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -357,7 +365,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!get_index_isclustered(indexOid))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
}
@@ -416,7 +423,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!RelationIsPopulated(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -424,7 +430,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -440,8 +445,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
rebuild_relation(OldHeap, indexOid, verbose);
/* NB: rebuild_relation does table_close() on OldHeap */
-
- pgstat_progress_end_command();
}
/*
@@ -1632,9 +1635,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
/* Cluster each relation in a separate transaction */
static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
{
ListCell *lc;
+ off_t childnum = 0;
/* Commit to get out of starting transaction */
PopActiveSnapshot();
@@ -1657,6 +1661,11 @@ cluster_multiple_rels(List *rvs, int options)
cluster_rel(rvtc->tableOid, rvtc->indexOid,
&cluster_params);
+ if (ispartitioned)
+ pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+ ++childnum);
+
+
PopActiveSnapshot();
CommitTransactionCommand();
}
diff --git a/src/backend/commands/vacuum.c b/src/backend/commands/vacuum.c
index c064352e23..05d52858b1 100644
--- a/src/backend/commands/vacuum.c
+++ b/src/backend/commands/vacuum.c
@@ -37,6 +37,7 @@
#include "catalog/pg_namespace.h"
#include "commands/cluster.h"
#include "commands/defrem.h"
+#include "commands/progress.h"
#include "commands/vacuum.h"
#include "miscadmin.h"
#include "nodes/makefuncs.h"
@@ -1937,7 +1938,11 @@ vacuum_rel(Oid relid, RangeVar *relation, VacuumParams *params)
cluster_params.options |= CLUOPT_VERBOSE;
/* VACUUM FULL is now a variant of CLUSTER; see cluster.c */
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, relid);
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
cluster_rel(relid, InvalidOid, &cluster_params);
+ pgstat_progress_end_command();
}
else
table_relation_vacuum(onerel, params, vac_strategy);
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
#define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS 5
#define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED 6
#define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT 7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL 17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE 18
/* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
#define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP 1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b1c9b7bdfe..c888415235 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1943,7 +1943,9 @@ pg_stat_progress_cluster| SELECT s.pid,
s.param5 AS heap_tuples_written,
s.param6 AS heap_blks_total,
s.param7 AS heap_blks_scanned,
- s.param8 AS index_rebuild_count
+ s.param8 AS index_rebuild_count,
+ s.param18 AS partitions_total,
+ s.param19 AS partitions_done
FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
LEFT JOIN pg_database d ON ((s.datid = d.oid)));
pg_stat_progress_copy| SELECT s.pid,
--
2.17.0
--fmvA4kSBHQVZhkR6
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v9-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"
^ permalink raw reply [nested|flat] 73+ messages in thread
* [PATCH v8 3/8] f! progress reporting..
@ 2021-02-09 02:45 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 73+ messages in thread
From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)
This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.
This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
src/backend/catalog/system_views.sql | 4 +-
src/backend/commands/cluster.c | 64 +++++++++++++++++-----------
src/include/commands/progress.h | 3 ++
src/test/regress/expected/rules.out | 4 +-
4 files changed, 47 insertions(+), 28 deletions(-)
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fa58afd9d7..9a3cb5f935 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1069,7 +1069,9 @@ CREATE VIEW pg_stat_progress_cluster AS
S.param5 AS heap_tuples_written,
S.param6 AS heap_blks_total,
S.param7 AS heap_blks_scanned,
- S.param8 AS index_rebuild_count
+ S.param8 AS index_rebuild_count,
+ S.param18 AS partitions_total,
+ S.param19 AS partitions_done
FROM pg_stat_get_progress_info('CLUSTER') AS S
LEFT JOIN pg_database D ON S.datid = D.oid;
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 9b6673867c..cb4fc350c6 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
static List *get_tables_to_cluster(MemoryContext cluster_context);
static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
/*---------------------------------------------------------------------------
@@ -188,16 +188,28 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
/* close relation, keep lock till commit */
table_close(rel, ShareUpdateExclusiveLock);
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
{
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ OidIsValid(indexOid) ? PROGRESS_CLUSTER_COMMAND_CLUSTER :
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
+
/* Do the job. */
cluster_rel(tableOid, indexOid, ¶ms);
+ pgstat_progress_end_command();
}
else
{
List *rvs;
MemoryContext cluster_context;
+ int64 progress_val[2];
+ const int progress_index[] = {
+ PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+ };
+
/* Refuse to hold strong locks in a user transaction */
PreventInTransactionBlock(isTopLevel, "CLUSTER");
@@ -206,7 +218,20 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
ALLOCSET_DEFAULT_SIZES);
rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
- cluster_multiple_rels(rvs, params.options);
+
+ /* Report vacuum full or cluster and number of partitions */
+ progress_val[0] = OidIsValid(indexOid) ?
+ PROGRESS_CLUSTER_COMMAND_CLUSTER :
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL;
+ progress_val[1] = list_length(rvs);
+// This is currently showing the total number of *tables*, including the
+// parent, and intermediate partitions, so the column should be renamed, or
+// we should count the number of leaf partitions
+
+ pgstat_progress_update_multi_param(2, progress_index, progress_val);
+
+ cluster_multiple_rels(rvs, params.options, true);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -245,7 +270,7 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
* cluster_context.
*/
rvs = get_tables_to_cluster(cluster_context);
- cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+ cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -282,14 +307,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* Check for user-requested abort. */
CHECK_FOR_INTERRUPTS();
- pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
- if (OidIsValid(indexOid))
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_CLUSTER);
- else
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
/*
* We grab exclusive access to the target rel and index for the duration
* of the transaction. (This is redundant for the single-transaction
@@ -300,10 +317,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* If the table has gone away, we can skip processing it */
if (!OldHeap)
- {
- pgstat_progress_end_command();
return;
- }
/*
* Since we may open a new transaction for each relation, we have to check
@@ -319,7 +333,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!pg_class_ownercheck(tableOid, GetUserId()))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -334,7 +347,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (RELATION_IS_OTHER_TEMP(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -346,7 +358,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -357,7 +368,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!get_index_isclustered(indexOid))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
}
@@ -416,7 +426,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!RelationIsPopulated(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -424,7 +433,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -440,8 +448,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
rebuild_relation(OldHeap, indexOid, verbose);
/* NB: rebuild_relation does table_close() on OldHeap */
-
- pgstat_progress_end_command();
}
/*
@@ -1366,7 +1372,7 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap,
ReindexParams reindex_params = {0};
int i;
- /* Report that we are now swapping relation files */
+ /* Report that we are now swapping relation files XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_SWAP_REL_FILES);
@@ -1417,13 +1423,13 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap,
else if (newrelpersistence == RELPERSISTENCE_PERMANENT)
reindex_flags |= REINDEX_REL_FORCE_INDEXES_PERMANENT;
- /* Report that we are now reindexing relations */
+ /* Report that we are now reindexing relations XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_REBUILD_INDEX);
reindex_relation(OIDOldHeap, reindex_flags, &reindex_params);
- /* Report that we are now doing clean up */
+ /* Report that we are now doing clean up XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_FINAL_CLEANUP);
@@ -1633,9 +1639,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
/* Cluster each relation in a separate transaction */
static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
{
ListCell *lc;
+ off_t childnum = 0;
/* Commit to get out of starting transaction */
PopActiveSnapshot();
@@ -1658,6 +1665,11 @@ cluster_multiple_rels(List *rvs, int options)
cluster_rel(rvtc->tableOid, rvtc->indexOid,
&cluster_params);
+ if (ispartitioned)
+ pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+ childnum++);
+
+
PopActiveSnapshot();
CommitTransactionCommand();
}
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
#define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS 5
#define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED 6
#define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT 7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL 17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE 18
/* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
#define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP 1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b632d9f2ea..d1fea532bc 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1941,7 +1941,9 @@ pg_stat_progress_cluster| SELECT s.pid,
s.param5 AS heap_tuples_written,
s.param6 AS heap_blks_total,
s.param7 AS heap_blks_scanned,
- s.param8 AS index_rebuild_count
+ s.param8 AS index_rebuild_count,
+ s.param18 AS partitions_total,
+ s.param19 AS partitions_done
FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
LEFT JOIN pg_database d ON ((s.datid = d.oid)));
pg_stat_progress_copy| SELECT s.pid,
--
2.17.0
--O5XBE6gyVG5Rl6Rj
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v8-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"
^ permalink raw reply [nested|flat] 73+ messages in thread
* [PATCH v9 3/8] f! progress reporting..
@ 2021-02-09 02:45 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 73+ messages in thread
From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)
This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.
This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
doc/src/sgml/monitoring.sgml | 21 +++++++++++
src/backend/catalog/system_views.sql | 4 +-
src/backend/commands/cluster.c | 55 ++++++++++++++++------------
src/backend/commands/vacuum.c | 5 +++
src/include/commands/progress.h | 3 ++
src/test/regress/expected/rules.out | 4 +-
6 files changed, 67 insertions(+), 25 deletions(-)
diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml
index 3513e127b7..dca47f6a48 100644
--- a/doc/src/sgml/monitoring.sgml
+++ b/doc/src/sgml/monitoring.sgml
@@ -6231,6 +6231,27 @@ SELECT pg_stat_get_backend_pid(s.backendid) AS pid,
is <literal>rebuilding index</literal>.
</para></entry>
</row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>total_partitions</structfield> <type>bigint</type>
+ </para>
+ When clustering a partitioned table, this column is set to the total
+ number of partitions to be clustered.
+ <para>
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>done_partitions</structfield> <type>bigint</type>
+ </para>
+ <para>
+ When clustering a partitioned table, this column is set to the number
+ of partitions which have been clustered.
+ </para></entry>
+ </row>
+
</tbody>
</tgroup>
</table>
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fc94a73a54..21c27f0437 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1070,7 +1070,9 @@ CREATE VIEW pg_stat_progress_cluster AS
S.param5 AS heap_tuples_written,
S.param6 AS heap_blks_total,
S.param7 AS heap_blks_scanned,
- S.param8 AS index_rebuild_count
+ S.param8 AS index_rebuild_count,
+ S.param18 AS partitions_total,
+ S.param19 AS partitions_done
FROM pg_stat_get_progress_info('CLUSTER') AS S
LEFT JOIN pg_database D ON S.datid = D.oid;
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 0c08ac56dc..7d8d1d8a69 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
static List *get_tables_to_cluster(MemoryContext cluster_context);
static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
/*---------------------------------------------------------------------------
@@ -188,16 +188,26 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
/* close relation, keep lock till commit */
table_close(rel, ShareUpdateExclusiveLock);
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
{
/* Do the job. */
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_CLUSTER);
cluster_rel(tableOid, indexOid, ¶ms);
+ pgstat_progress_end_command();
}
else
{
List *rvs;
MemoryContext cluster_context;
+ int64 progress_val[2];
+ const int progress_index[2] = {
+ PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+ };
+
/* Refuse to hold strong locks in a user transaction */
PreventInTransactionBlock(isTopLevel, "CLUSTER");
@@ -206,7 +216,15 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
ALLOCSET_DEFAULT_SIZES);
rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
- cluster_multiple_rels(rvs, params.options);
+
+ /* Report command and number of partitions */
+ progress_val[0] = PROGRESS_CLUSTER_COMMAND_CLUSTER;
+ /* XXX: rvs includes the parent, which is not a "partition" */
+ progress_val[1] = list_length(rvs);
+
+ pgstat_progress_update_multi_param(2, progress_index, progress_val);
+ cluster_multiple_rels(rvs, params.options, true);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -245,7 +263,11 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
* cluster_context.
*/
rvs = get_tables_to_cluster(cluster_context);
- cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_CLUSTER);
+ cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -282,14 +304,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* Check for user-requested abort. */
CHECK_FOR_INTERRUPTS();
- pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
- if (OidIsValid(indexOid))
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_CLUSTER);
- else
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
/*
* We grab exclusive access to the target rel and index for the duration
* of the transaction. (This is redundant for the single-transaction
@@ -300,10 +314,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* If the table has gone away, we can skip processing it */
if (!OldHeap)
- {
- pgstat_progress_end_command();
return;
- }
/*
* Since we may open a new transaction for each relation, we have to check
@@ -319,7 +330,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!pg_class_ownercheck(tableOid, GetUserId()))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -334,7 +344,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (RELATION_IS_OTHER_TEMP(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -346,7 +355,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -357,7 +365,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!get_index_isclustered(indexOid))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
}
@@ -416,7 +423,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!RelationIsPopulated(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -424,7 +430,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -440,8 +445,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
rebuild_relation(OldHeap, indexOid, verbose);
/* NB: rebuild_relation does table_close() on OldHeap */
-
- pgstat_progress_end_command();
}
/*
@@ -1632,9 +1635,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
/* Cluster each relation in a separate transaction */
static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
{
ListCell *lc;
+ off_t childnum = 0;
/* Commit to get out of starting transaction */
PopActiveSnapshot();
@@ -1657,6 +1661,11 @@ cluster_multiple_rels(List *rvs, int options)
cluster_rel(rvtc->tableOid, rvtc->indexOid,
&cluster_params);
+ if (ispartitioned)
+ pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+ ++childnum);
+
+
PopActiveSnapshot();
CommitTransactionCommand();
}
diff --git a/src/backend/commands/vacuum.c b/src/backend/commands/vacuum.c
index c064352e23..05d52858b1 100644
--- a/src/backend/commands/vacuum.c
+++ b/src/backend/commands/vacuum.c
@@ -37,6 +37,7 @@
#include "catalog/pg_namespace.h"
#include "commands/cluster.h"
#include "commands/defrem.h"
+#include "commands/progress.h"
#include "commands/vacuum.h"
#include "miscadmin.h"
#include "nodes/makefuncs.h"
@@ -1937,7 +1938,11 @@ vacuum_rel(Oid relid, RangeVar *relation, VacuumParams *params)
cluster_params.options |= CLUOPT_VERBOSE;
/* VACUUM FULL is now a variant of CLUSTER; see cluster.c */
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, relid);
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
cluster_rel(relid, InvalidOid, &cluster_params);
+ pgstat_progress_end_command();
}
else
table_relation_vacuum(onerel, params, vac_strategy);
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
#define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS 5
#define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED 6
#define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT 7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL 17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE 18
/* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
#define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP 1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b1c9b7bdfe..c888415235 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1943,7 +1943,9 @@ pg_stat_progress_cluster| SELECT s.pid,
s.param5 AS heap_tuples_written,
s.param6 AS heap_blks_total,
s.param7 AS heap_blks_scanned,
- s.param8 AS index_rebuild_count
+ s.param8 AS index_rebuild_count,
+ s.param18 AS partitions_total,
+ s.param19 AS partitions_done
FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
LEFT JOIN pg_database d ON ((s.datid = d.oid)));
pg_stat_progress_copy| SELECT s.pid,
--
2.17.0
--fmvA4kSBHQVZhkR6
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v9-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"
^ permalink raw reply [nested|flat] 73+ messages in thread
* [PATCH v9 3/8] f! progress reporting..
@ 2021-02-09 02:45 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 73+ messages in thread
From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)
This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.
This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
doc/src/sgml/monitoring.sgml | 21 +++++++++++
src/backend/catalog/system_views.sql | 4 +-
src/backend/commands/cluster.c | 55 ++++++++++++++++------------
src/backend/commands/vacuum.c | 5 +++
src/include/commands/progress.h | 3 ++
src/test/regress/expected/rules.out | 4 +-
6 files changed, 67 insertions(+), 25 deletions(-)
diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml
index 3513e127b7..dca47f6a48 100644
--- a/doc/src/sgml/monitoring.sgml
+++ b/doc/src/sgml/monitoring.sgml
@@ -6231,6 +6231,27 @@ SELECT pg_stat_get_backend_pid(s.backendid) AS pid,
is <literal>rebuilding index</literal>.
</para></entry>
</row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>total_partitions</structfield> <type>bigint</type>
+ </para>
+ When clustering a partitioned table, this column is set to the total
+ number of partitions to be clustered.
+ <para>
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>done_partitions</structfield> <type>bigint</type>
+ </para>
+ <para>
+ When clustering a partitioned table, this column is set to the number
+ of partitions which have been clustered.
+ </para></entry>
+ </row>
+
</tbody>
</tgroup>
</table>
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fc94a73a54..21c27f0437 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1070,7 +1070,9 @@ CREATE VIEW pg_stat_progress_cluster AS
S.param5 AS heap_tuples_written,
S.param6 AS heap_blks_total,
S.param7 AS heap_blks_scanned,
- S.param8 AS index_rebuild_count
+ S.param8 AS index_rebuild_count,
+ S.param18 AS partitions_total,
+ S.param19 AS partitions_done
FROM pg_stat_get_progress_info('CLUSTER') AS S
LEFT JOIN pg_database D ON S.datid = D.oid;
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 0c08ac56dc..7d8d1d8a69 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
static List *get_tables_to_cluster(MemoryContext cluster_context);
static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
/*---------------------------------------------------------------------------
@@ -188,16 +188,26 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
/* close relation, keep lock till commit */
table_close(rel, ShareUpdateExclusiveLock);
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
{
/* Do the job. */
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_CLUSTER);
cluster_rel(tableOid, indexOid, ¶ms);
+ pgstat_progress_end_command();
}
else
{
List *rvs;
MemoryContext cluster_context;
+ int64 progress_val[2];
+ const int progress_index[2] = {
+ PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+ };
+
/* Refuse to hold strong locks in a user transaction */
PreventInTransactionBlock(isTopLevel, "CLUSTER");
@@ -206,7 +216,15 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
ALLOCSET_DEFAULT_SIZES);
rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
- cluster_multiple_rels(rvs, params.options);
+
+ /* Report command and number of partitions */
+ progress_val[0] = PROGRESS_CLUSTER_COMMAND_CLUSTER;
+ /* XXX: rvs includes the parent, which is not a "partition" */
+ progress_val[1] = list_length(rvs);
+
+ pgstat_progress_update_multi_param(2, progress_index, progress_val);
+ cluster_multiple_rels(rvs, params.options, true);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -245,7 +263,11 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
* cluster_context.
*/
rvs = get_tables_to_cluster(cluster_context);
- cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_CLUSTER);
+ cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -282,14 +304,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* Check for user-requested abort. */
CHECK_FOR_INTERRUPTS();
- pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
- if (OidIsValid(indexOid))
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_CLUSTER);
- else
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
/*
* We grab exclusive access to the target rel and index for the duration
* of the transaction. (This is redundant for the single-transaction
@@ -300,10 +314,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* If the table has gone away, we can skip processing it */
if (!OldHeap)
- {
- pgstat_progress_end_command();
return;
- }
/*
* Since we may open a new transaction for each relation, we have to check
@@ -319,7 +330,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!pg_class_ownercheck(tableOid, GetUserId()))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -334,7 +344,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (RELATION_IS_OTHER_TEMP(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -346,7 +355,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -357,7 +365,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!get_index_isclustered(indexOid))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
}
@@ -416,7 +423,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!RelationIsPopulated(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -424,7 +430,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -440,8 +445,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
rebuild_relation(OldHeap, indexOid, verbose);
/* NB: rebuild_relation does table_close() on OldHeap */
-
- pgstat_progress_end_command();
}
/*
@@ -1632,9 +1635,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
/* Cluster each relation in a separate transaction */
static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
{
ListCell *lc;
+ off_t childnum = 0;
/* Commit to get out of starting transaction */
PopActiveSnapshot();
@@ -1657,6 +1661,11 @@ cluster_multiple_rels(List *rvs, int options)
cluster_rel(rvtc->tableOid, rvtc->indexOid,
&cluster_params);
+ if (ispartitioned)
+ pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+ ++childnum);
+
+
PopActiveSnapshot();
CommitTransactionCommand();
}
diff --git a/src/backend/commands/vacuum.c b/src/backend/commands/vacuum.c
index c064352e23..05d52858b1 100644
--- a/src/backend/commands/vacuum.c
+++ b/src/backend/commands/vacuum.c
@@ -37,6 +37,7 @@
#include "catalog/pg_namespace.h"
#include "commands/cluster.h"
#include "commands/defrem.h"
+#include "commands/progress.h"
#include "commands/vacuum.h"
#include "miscadmin.h"
#include "nodes/makefuncs.h"
@@ -1937,7 +1938,11 @@ vacuum_rel(Oid relid, RangeVar *relation, VacuumParams *params)
cluster_params.options |= CLUOPT_VERBOSE;
/* VACUUM FULL is now a variant of CLUSTER; see cluster.c */
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, relid);
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
cluster_rel(relid, InvalidOid, &cluster_params);
+ pgstat_progress_end_command();
}
else
table_relation_vacuum(onerel, params, vac_strategy);
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
#define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS 5
#define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED 6
#define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT 7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL 17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE 18
/* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
#define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP 1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b1c9b7bdfe..c888415235 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1943,7 +1943,9 @@ pg_stat_progress_cluster| SELECT s.pid,
s.param5 AS heap_tuples_written,
s.param6 AS heap_blks_total,
s.param7 AS heap_blks_scanned,
- s.param8 AS index_rebuild_count
+ s.param8 AS index_rebuild_count,
+ s.param18 AS partitions_total,
+ s.param19 AS partitions_done
FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
LEFT JOIN pg_database d ON ((s.datid = d.oid)));
pg_stat_progress_copy| SELECT s.pid,
--
2.17.0
--fmvA4kSBHQVZhkR6
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v9-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"
^ permalink raw reply [nested|flat] 73+ messages in thread
* [PATCH v8 3/8] f! progress reporting..
@ 2021-02-09 02:45 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 73+ messages in thread
From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)
This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.
This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
src/backend/catalog/system_views.sql | 4 +-
src/backend/commands/cluster.c | 64 +++++++++++++++++-----------
src/include/commands/progress.h | 3 ++
src/test/regress/expected/rules.out | 4 +-
4 files changed, 47 insertions(+), 28 deletions(-)
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fa58afd9d7..9a3cb5f935 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1069,7 +1069,9 @@ CREATE VIEW pg_stat_progress_cluster AS
S.param5 AS heap_tuples_written,
S.param6 AS heap_blks_total,
S.param7 AS heap_blks_scanned,
- S.param8 AS index_rebuild_count
+ S.param8 AS index_rebuild_count,
+ S.param18 AS partitions_total,
+ S.param19 AS partitions_done
FROM pg_stat_get_progress_info('CLUSTER') AS S
LEFT JOIN pg_database D ON S.datid = D.oid;
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 9b6673867c..cb4fc350c6 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
static List *get_tables_to_cluster(MemoryContext cluster_context);
static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
/*---------------------------------------------------------------------------
@@ -188,16 +188,28 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
/* close relation, keep lock till commit */
table_close(rel, ShareUpdateExclusiveLock);
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
{
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ OidIsValid(indexOid) ? PROGRESS_CLUSTER_COMMAND_CLUSTER :
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
+
/* Do the job. */
cluster_rel(tableOid, indexOid, ¶ms);
+ pgstat_progress_end_command();
}
else
{
List *rvs;
MemoryContext cluster_context;
+ int64 progress_val[2];
+ const int progress_index[] = {
+ PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+ };
+
/* Refuse to hold strong locks in a user transaction */
PreventInTransactionBlock(isTopLevel, "CLUSTER");
@@ -206,7 +218,20 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
ALLOCSET_DEFAULT_SIZES);
rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
- cluster_multiple_rels(rvs, params.options);
+
+ /* Report vacuum full or cluster and number of partitions */
+ progress_val[0] = OidIsValid(indexOid) ?
+ PROGRESS_CLUSTER_COMMAND_CLUSTER :
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL;
+ progress_val[1] = list_length(rvs);
+// This is currently showing the total number of *tables*, including the
+// parent, and intermediate partitions, so the column should be renamed, or
+// we should count the number of leaf partitions
+
+ pgstat_progress_update_multi_param(2, progress_index, progress_val);
+
+ cluster_multiple_rels(rvs, params.options, true);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -245,7 +270,7 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
* cluster_context.
*/
rvs = get_tables_to_cluster(cluster_context);
- cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+ cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -282,14 +307,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* Check for user-requested abort. */
CHECK_FOR_INTERRUPTS();
- pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
- if (OidIsValid(indexOid))
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_CLUSTER);
- else
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
/*
* We grab exclusive access to the target rel and index for the duration
* of the transaction. (This is redundant for the single-transaction
@@ -300,10 +317,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* If the table has gone away, we can skip processing it */
if (!OldHeap)
- {
- pgstat_progress_end_command();
return;
- }
/*
* Since we may open a new transaction for each relation, we have to check
@@ -319,7 +333,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!pg_class_ownercheck(tableOid, GetUserId()))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -334,7 +347,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (RELATION_IS_OTHER_TEMP(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -346,7 +358,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -357,7 +368,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!get_index_isclustered(indexOid))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
}
@@ -416,7 +426,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!RelationIsPopulated(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -424,7 +433,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -440,8 +448,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
rebuild_relation(OldHeap, indexOid, verbose);
/* NB: rebuild_relation does table_close() on OldHeap */
-
- pgstat_progress_end_command();
}
/*
@@ -1366,7 +1372,7 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap,
ReindexParams reindex_params = {0};
int i;
- /* Report that we are now swapping relation files */
+ /* Report that we are now swapping relation files XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_SWAP_REL_FILES);
@@ -1417,13 +1423,13 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap,
else if (newrelpersistence == RELPERSISTENCE_PERMANENT)
reindex_flags |= REINDEX_REL_FORCE_INDEXES_PERMANENT;
- /* Report that we are now reindexing relations */
+ /* Report that we are now reindexing relations XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_REBUILD_INDEX);
reindex_relation(OIDOldHeap, reindex_flags, &reindex_params);
- /* Report that we are now doing clean up */
+ /* Report that we are now doing clean up XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_FINAL_CLEANUP);
@@ -1633,9 +1639,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
/* Cluster each relation in a separate transaction */
static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
{
ListCell *lc;
+ off_t childnum = 0;
/* Commit to get out of starting transaction */
PopActiveSnapshot();
@@ -1658,6 +1665,11 @@ cluster_multiple_rels(List *rvs, int options)
cluster_rel(rvtc->tableOid, rvtc->indexOid,
&cluster_params);
+ if (ispartitioned)
+ pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+ childnum++);
+
+
PopActiveSnapshot();
CommitTransactionCommand();
}
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
#define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS 5
#define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED 6
#define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT 7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL 17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE 18
/* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
#define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP 1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b632d9f2ea..d1fea532bc 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1941,7 +1941,9 @@ pg_stat_progress_cluster| SELECT s.pid,
s.param5 AS heap_tuples_written,
s.param6 AS heap_blks_total,
s.param7 AS heap_blks_scanned,
- s.param8 AS index_rebuild_count
+ s.param8 AS index_rebuild_count,
+ s.param18 AS partitions_total,
+ s.param19 AS partitions_done
FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
LEFT JOIN pg_database d ON ((s.datid = d.oid)));
pg_stat_progress_copy| SELECT s.pid,
--
2.17.0
--O5XBE6gyVG5Rl6Rj
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v8-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"
^ permalink raw reply [nested|flat] 73+ messages in thread
* [PATCH v9 3/8] f! progress reporting..
@ 2021-02-09 02:45 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 73+ messages in thread
From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)
This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.
This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
doc/src/sgml/monitoring.sgml | 21 +++++++++++
src/backend/catalog/system_views.sql | 4 +-
src/backend/commands/cluster.c | 55 ++++++++++++++++------------
src/backend/commands/vacuum.c | 5 +++
src/include/commands/progress.h | 3 ++
src/test/regress/expected/rules.out | 4 +-
6 files changed, 67 insertions(+), 25 deletions(-)
diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml
index 3513e127b7..dca47f6a48 100644
--- a/doc/src/sgml/monitoring.sgml
+++ b/doc/src/sgml/monitoring.sgml
@@ -6231,6 +6231,27 @@ SELECT pg_stat_get_backend_pid(s.backendid) AS pid,
is <literal>rebuilding index</literal>.
</para></entry>
</row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>total_partitions</structfield> <type>bigint</type>
+ </para>
+ When clustering a partitioned table, this column is set to the total
+ number of partitions to be clustered.
+ <para>
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>done_partitions</structfield> <type>bigint</type>
+ </para>
+ <para>
+ When clustering a partitioned table, this column is set to the number
+ of partitions which have been clustered.
+ </para></entry>
+ </row>
+
</tbody>
</tgroup>
</table>
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fc94a73a54..21c27f0437 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1070,7 +1070,9 @@ CREATE VIEW pg_stat_progress_cluster AS
S.param5 AS heap_tuples_written,
S.param6 AS heap_blks_total,
S.param7 AS heap_blks_scanned,
- S.param8 AS index_rebuild_count
+ S.param8 AS index_rebuild_count,
+ S.param18 AS partitions_total,
+ S.param19 AS partitions_done
FROM pg_stat_get_progress_info('CLUSTER') AS S
LEFT JOIN pg_database D ON S.datid = D.oid;
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 0c08ac56dc..7d8d1d8a69 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
static List *get_tables_to_cluster(MemoryContext cluster_context);
static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
/*---------------------------------------------------------------------------
@@ -188,16 +188,26 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
/* close relation, keep lock till commit */
table_close(rel, ShareUpdateExclusiveLock);
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
{
/* Do the job. */
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_CLUSTER);
cluster_rel(tableOid, indexOid, ¶ms);
+ pgstat_progress_end_command();
}
else
{
List *rvs;
MemoryContext cluster_context;
+ int64 progress_val[2];
+ const int progress_index[2] = {
+ PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+ };
+
/* Refuse to hold strong locks in a user transaction */
PreventInTransactionBlock(isTopLevel, "CLUSTER");
@@ -206,7 +216,15 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
ALLOCSET_DEFAULT_SIZES);
rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
- cluster_multiple_rels(rvs, params.options);
+
+ /* Report command and number of partitions */
+ progress_val[0] = PROGRESS_CLUSTER_COMMAND_CLUSTER;
+ /* XXX: rvs includes the parent, which is not a "partition" */
+ progress_val[1] = list_length(rvs);
+
+ pgstat_progress_update_multi_param(2, progress_index, progress_val);
+ cluster_multiple_rels(rvs, params.options, true);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -245,7 +263,11 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
* cluster_context.
*/
rvs = get_tables_to_cluster(cluster_context);
- cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_CLUSTER);
+ cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -282,14 +304,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* Check for user-requested abort. */
CHECK_FOR_INTERRUPTS();
- pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
- if (OidIsValid(indexOid))
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_CLUSTER);
- else
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
/*
* We grab exclusive access to the target rel and index for the duration
* of the transaction. (This is redundant for the single-transaction
@@ -300,10 +314,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* If the table has gone away, we can skip processing it */
if (!OldHeap)
- {
- pgstat_progress_end_command();
return;
- }
/*
* Since we may open a new transaction for each relation, we have to check
@@ -319,7 +330,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!pg_class_ownercheck(tableOid, GetUserId()))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -334,7 +344,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (RELATION_IS_OTHER_TEMP(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -346,7 +355,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -357,7 +365,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!get_index_isclustered(indexOid))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
}
@@ -416,7 +423,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!RelationIsPopulated(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -424,7 +430,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -440,8 +445,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
rebuild_relation(OldHeap, indexOid, verbose);
/* NB: rebuild_relation does table_close() on OldHeap */
-
- pgstat_progress_end_command();
}
/*
@@ -1632,9 +1635,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
/* Cluster each relation in a separate transaction */
static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
{
ListCell *lc;
+ off_t childnum = 0;
/* Commit to get out of starting transaction */
PopActiveSnapshot();
@@ -1657,6 +1661,11 @@ cluster_multiple_rels(List *rvs, int options)
cluster_rel(rvtc->tableOid, rvtc->indexOid,
&cluster_params);
+ if (ispartitioned)
+ pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+ ++childnum);
+
+
PopActiveSnapshot();
CommitTransactionCommand();
}
diff --git a/src/backend/commands/vacuum.c b/src/backend/commands/vacuum.c
index c064352e23..05d52858b1 100644
--- a/src/backend/commands/vacuum.c
+++ b/src/backend/commands/vacuum.c
@@ -37,6 +37,7 @@
#include "catalog/pg_namespace.h"
#include "commands/cluster.h"
#include "commands/defrem.h"
+#include "commands/progress.h"
#include "commands/vacuum.h"
#include "miscadmin.h"
#include "nodes/makefuncs.h"
@@ -1937,7 +1938,11 @@ vacuum_rel(Oid relid, RangeVar *relation, VacuumParams *params)
cluster_params.options |= CLUOPT_VERBOSE;
/* VACUUM FULL is now a variant of CLUSTER; see cluster.c */
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, relid);
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
cluster_rel(relid, InvalidOid, &cluster_params);
+ pgstat_progress_end_command();
}
else
table_relation_vacuum(onerel, params, vac_strategy);
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
#define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS 5
#define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED 6
#define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT 7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL 17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE 18
/* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
#define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP 1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b1c9b7bdfe..c888415235 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1943,7 +1943,9 @@ pg_stat_progress_cluster| SELECT s.pid,
s.param5 AS heap_tuples_written,
s.param6 AS heap_blks_total,
s.param7 AS heap_blks_scanned,
- s.param8 AS index_rebuild_count
+ s.param8 AS index_rebuild_count,
+ s.param18 AS partitions_total,
+ s.param19 AS partitions_done
FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
LEFT JOIN pg_database d ON ((s.datid = d.oid)));
pg_stat_progress_copy| SELECT s.pid,
--
2.17.0
--fmvA4kSBHQVZhkR6
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v9-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"
^ permalink raw reply [nested|flat] 73+ messages in thread
* [PATCH v9 3/8] f! progress reporting..
@ 2021-02-09 02:45 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 73+ messages in thread
From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)
This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.
This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
doc/src/sgml/monitoring.sgml | 21 +++++++++++
src/backend/catalog/system_views.sql | 4 +-
src/backend/commands/cluster.c | 55 ++++++++++++++++------------
src/backend/commands/vacuum.c | 5 +++
src/include/commands/progress.h | 3 ++
src/test/regress/expected/rules.out | 4 +-
6 files changed, 67 insertions(+), 25 deletions(-)
diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml
index 3513e127b7..dca47f6a48 100644
--- a/doc/src/sgml/monitoring.sgml
+++ b/doc/src/sgml/monitoring.sgml
@@ -6231,6 +6231,27 @@ SELECT pg_stat_get_backend_pid(s.backendid) AS pid,
is <literal>rebuilding index</literal>.
</para></entry>
</row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>total_partitions</structfield> <type>bigint</type>
+ </para>
+ When clustering a partitioned table, this column is set to the total
+ number of partitions to be clustered.
+ <para>
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>done_partitions</structfield> <type>bigint</type>
+ </para>
+ <para>
+ When clustering a partitioned table, this column is set to the number
+ of partitions which have been clustered.
+ </para></entry>
+ </row>
+
</tbody>
</tgroup>
</table>
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fc94a73a54..21c27f0437 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1070,7 +1070,9 @@ CREATE VIEW pg_stat_progress_cluster AS
S.param5 AS heap_tuples_written,
S.param6 AS heap_blks_total,
S.param7 AS heap_blks_scanned,
- S.param8 AS index_rebuild_count
+ S.param8 AS index_rebuild_count,
+ S.param18 AS partitions_total,
+ S.param19 AS partitions_done
FROM pg_stat_get_progress_info('CLUSTER') AS S
LEFT JOIN pg_database D ON S.datid = D.oid;
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 0c08ac56dc..7d8d1d8a69 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
static List *get_tables_to_cluster(MemoryContext cluster_context);
static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
/*---------------------------------------------------------------------------
@@ -188,16 +188,26 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
/* close relation, keep lock till commit */
table_close(rel, ShareUpdateExclusiveLock);
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
{
/* Do the job. */
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_CLUSTER);
cluster_rel(tableOid, indexOid, ¶ms);
+ pgstat_progress_end_command();
}
else
{
List *rvs;
MemoryContext cluster_context;
+ int64 progress_val[2];
+ const int progress_index[2] = {
+ PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+ };
+
/* Refuse to hold strong locks in a user transaction */
PreventInTransactionBlock(isTopLevel, "CLUSTER");
@@ -206,7 +216,15 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
ALLOCSET_DEFAULT_SIZES);
rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
- cluster_multiple_rels(rvs, params.options);
+
+ /* Report command and number of partitions */
+ progress_val[0] = PROGRESS_CLUSTER_COMMAND_CLUSTER;
+ /* XXX: rvs includes the parent, which is not a "partition" */
+ progress_val[1] = list_length(rvs);
+
+ pgstat_progress_update_multi_param(2, progress_index, progress_val);
+ cluster_multiple_rels(rvs, params.options, true);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -245,7 +263,11 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
* cluster_context.
*/
rvs = get_tables_to_cluster(cluster_context);
- cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_CLUSTER);
+ cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -282,14 +304,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* Check for user-requested abort. */
CHECK_FOR_INTERRUPTS();
- pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
- if (OidIsValid(indexOid))
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_CLUSTER);
- else
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
/*
* We grab exclusive access to the target rel and index for the duration
* of the transaction. (This is redundant for the single-transaction
@@ -300,10 +314,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* If the table has gone away, we can skip processing it */
if (!OldHeap)
- {
- pgstat_progress_end_command();
return;
- }
/*
* Since we may open a new transaction for each relation, we have to check
@@ -319,7 +330,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!pg_class_ownercheck(tableOid, GetUserId()))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -334,7 +344,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (RELATION_IS_OTHER_TEMP(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -346,7 +355,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -357,7 +365,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!get_index_isclustered(indexOid))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
}
@@ -416,7 +423,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!RelationIsPopulated(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -424,7 +430,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -440,8 +445,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
rebuild_relation(OldHeap, indexOid, verbose);
/* NB: rebuild_relation does table_close() on OldHeap */
-
- pgstat_progress_end_command();
}
/*
@@ -1632,9 +1635,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
/* Cluster each relation in a separate transaction */
static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
{
ListCell *lc;
+ off_t childnum = 0;
/* Commit to get out of starting transaction */
PopActiveSnapshot();
@@ -1657,6 +1661,11 @@ cluster_multiple_rels(List *rvs, int options)
cluster_rel(rvtc->tableOid, rvtc->indexOid,
&cluster_params);
+ if (ispartitioned)
+ pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+ ++childnum);
+
+
PopActiveSnapshot();
CommitTransactionCommand();
}
diff --git a/src/backend/commands/vacuum.c b/src/backend/commands/vacuum.c
index c064352e23..05d52858b1 100644
--- a/src/backend/commands/vacuum.c
+++ b/src/backend/commands/vacuum.c
@@ -37,6 +37,7 @@
#include "catalog/pg_namespace.h"
#include "commands/cluster.h"
#include "commands/defrem.h"
+#include "commands/progress.h"
#include "commands/vacuum.h"
#include "miscadmin.h"
#include "nodes/makefuncs.h"
@@ -1937,7 +1938,11 @@ vacuum_rel(Oid relid, RangeVar *relation, VacuumParams *params)
cluster_params.options |= CLUOPT_VERBOSE;
/* VACUUM FULL is now a variant of CLUSTER; see cluster.c */
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, relid);
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
cluster_rel(relid, InvalidOid, &cluster_params);
+ pgstat_progress_end_command();
}
else
table_relation_vacuum(onerel, params, vac_strategy);
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
#define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS 5
#define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED 6
#define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT 7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL 17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE 18
/* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
#define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP 1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b1c9b7bdfe..c888415235 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1943,7 +1943,9 @@ pg_stat_progress_cluster| SELECT s.pid,
s.param5 AS heap_tuples_written,
s.param6 AS heap_blks_total,
s.param7 AS heap_blks_scanned,
- s.param8 AS index_rebuild_count
+ s.param8 AS index_rebuild_count,
+ s.param18 AS partitions_total,
+ s.param19 AS partitions_done
FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
LEFT JOIN pg_database d ON ((s.datid = d.oid)));
pg_stat_progress_copy| SELECT s.pid,
--
2.17.0
--fmvA4kSBHQVZhkR6
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v9-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"
^ permalink raw reply [nested|flat] 73+ messages in thread
* [PATCH v8 3/8] f! progress reporting..
@ 2021-02-09 02:45 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 73+ messages in thread
From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)
This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.
This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
src/backend/catalog/system_views.sql | 4 +-
src/backend/commands/cluster.c | 64 +++++++++++++++++-----------
src/include/commands/progress.h | 3 ++
src/test/regress/expected/rules.out | 4 +-
4 files changed, 47 insertions(+), 28 deletions(-)
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fa58afd9d7..9a3cb5f935 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1069,7 +1069,9 @@ CREATE VIEW pg_stat_progress_cluster AS
S.param5 AS heap_tuples_written,
S.param6 AS heap_blks_total,
S.param7 AS heap_blks_scanned,
- S.param8 AS index_rebuild_count
+ S.param8 AS index_rebuild_count,
+ S.param18 AS partitions_total,
+ S.param19 AS partitions_done
FROM pg_stat_get_progress_info('CLUSTER') AS S
LEFT JOIN pg_database D ON S.datid = D.oid;
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 9b6673867c..cb4fc350c6 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
static List *get_tables_to_cluster(MemoryContext cluster_context);
static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
/*---------------------------------------------------------------------------
@@ -188,16 +188,28 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
/* close relation, keep lock till commit */
table_close(rel, ShareUpdateExclusiveLock);
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
{
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ OidIsValid(indexOid) ? PROGRESS_CLUSTER_COMMAND_CLUSTER :
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
+
/* Do the job. */
cluster_rel(tableOid, indexOid, ¶ms);
+ pgstat_progress_end_command();
}
else
{
List *rvs;
MemoryContext cluster_context;
+ int64 progress_val[2];
+ const int progress_index[] = {
+ PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+ };
+
/* Refuse to hold strong locks in a user transaction */
PreventInTransactionBlock(isTopLevel, "CLUSTER");
@@ -206,7 +218,20 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
ALLOCSET_DEFAULT_SIZES);
rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
- cluster_multiple_rels(rvs, params.options);
+
+ /* Report vacuum full or cluster and number of partitions */
+ progress_val[0] = OidIsValid(indexOid) ?
+ PROGRESS_CLUSTER_COMMAND_CLUSTER :
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL;
+ progress_val[1] = list_length(rvs);
+// This is currently showing the total number of *tables*, including the
+// parent, and intermediate partitions, so the column should be renamed, or
+// we should count the number of leaf partitions
+
+ pgstat_progress_update_multi_param(2, progress_index, progress_val);
+
+ cluster_multiple_rels(rvs, params.options, true);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -245,7 +270,7 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
* cluster_context.
*/
rvs = get_tables_to_cluster(cluster_context);
- cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+ cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -282,14 +307,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* Check for user-requested abort. */
CHECK_FOR_INTERRUPTS();
- pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
- if (OidIsValid(indexOid))
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_CLUSTER);
- else
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
/*
* We grab exclusive access to the target rel and index for the duration
* of the transaction. (This is redundant for the single-transaction
@@ -300,10 +317,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* If the table has gone away, we can skip processing it */
if (!OldHeap)
- {
- pgstat_progress_end_command();
return;
- }
/*
* Since we may open a new transaction for each relation, we have to check
@@ -319,7 +333,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!pg_class_ownercheck(tableOid, GetUserId()))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -334,7 +347,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (RELATION_IS_OTHER_TEMP(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -346,7 +358,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -357,7 +368,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!get_index_isclustered(indexOid))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
}
@@ -416,7 +426,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!RelationIsPopulated(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -424,7 +433,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -440,8 +448,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
rebuild_relation(OldHeap, indexOid, verbose);
/* NB: rebuild_relation does table_close() on OldHeap */
-
- pgstat_progress_end_command();
}
/*
@@ -1366,7 +1372,7 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap,
ReindexParams reindex_params = {0};
int i;
- /* Report that we are now swapping relation files */
+ /* Report that we are now swapping relation files XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_SWAP_REL_FILES);
@@ -1417,13 +1423,13 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap,
else if (newrelpersistence == RELPERSISTENCE_PERMANENT)
reindex_flags |= REINDEX_REL_FORCE_INDEXES_PERMANENT;
- /* Report that we are now reindexing relations */
+ /* Report that we are now reindexing relations XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_REBUILD_INDEX);
reindex_relation(OIDOldHeap, reindex_flags, &reindex_params);
- /* Report that we are now doing clean up */
+ /* Report that we are now doing clean up XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_FINAL_CLEANUP);
@@ -1633,9 +1639,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
/* Cluster each relation in a separate transaction */
static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
{
ListCell *lc;
+ off_t childnum = 0;
/* Commit to get out of starting transaction */
PopActiveSnapshot();
@@ -1658,6 +1665,11 @@ cluster_multiple_rels(List *rvs, int options)
cluster_rel(rvtc->tableOid, rvtc->indexOid,
&cluster_params);
+ if (ispartitioned)
+ pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+ childnum++);
+
+
PopActiveSnapshot();
CommitTransactionCommand();
}
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
#define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS 5
#define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED 6
#define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT 7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL 17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE 18
/* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
#define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP 1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b632d9f2ea..d1fea532bc 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1941,7 +1941,9 @@ pg_stat_progress_cluster| SELECT s.pid,
s.param5 AS heap_tuples_written,
s.param6 AS heap_blks_total,
s.param7 AS heap_blks_scanned,
- s.param8 AS index_rebuild_count
+ s.param8 AS index_rebuild_count,
+ s.param18 AS partitions_total,
+ s.param19 AS partitions_done
FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
LEFT JOIN pg_database d ON ((s.datid = d.oid)));
pg_stat_progress_copy| SELECT s.pid,
--
2.17.0
--O5XBE6gyVG5Rl6Rj
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v8-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"
^ permalink raw reply [nested|flat] 73+ messages in thread
* [PATCH v9 3/8] f! progress reporting..
@ 2021-02-09 02:45 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 73+ messages in thread
From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)
This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.
This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
doc/src/sgml/monitoring.sgml | 21 +++++++++++
src/backend/catalog/system_views.sql | 4 +-
src/backend/commands/cluster.c | 55 ++++++++++++++++------------
src/backend/commands/vacuum.c | 5 +++
src/include/commands/progress.h | 3 ++
src/test/regress/expected/rules.out | 4 +-
6 files changed, 67 insertions(+), 25 deletions(-)
diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml
index 3513e127b7..dca47f6a48 100644
--- a/doc/src/sgml/monitoring.sgml
+++ b/doc/src/sgml/monitoring.sgml
@@ -6231,6 +6231,27 @@ SELECT pg_stat_get_backend_pid(s.backendid) AS pid,
is <literal>rebuilding index</literal>.
</para></entry>
</row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>total_partitions</structfield> <type>bigint</type>
+ </para>
+ When clustering a partitioned table, this column is set to the total
+ number of partitions to be clustered.
+ <para>
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>done_partitions</structfield> <type>bigint</type>
+ </para>
+ <para>
+ When clustering a partitioned table, this column is set to the number
+ of partitions which have been clustered.
+ </para></entry>
+ </row>
+
</tbody>
</tgroup>
</table>
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fc94a73a54..21c27f0437 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1070,7 +1070,9 @@ CREATE VIEW pg_stat_progress_cluster AS
S.param5 AS heap_tuples_written,
S.param6 AS heap_blks_total,
S.param7 AS heap_blks_scanned,
- S.param8 AS index_rebuild_count
+ S.param8 AS index_rebuild_count,
+ S.param18 AS partitions_total,
+ S.param19 AS partitions_done
FROM pg_stat_get_progress_info('CLUSTER') AS S
LEFT JOIN pg_database D ON S.datid = D.oid;
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 0c08ac56dc..7d8d1d8a69 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
static List *get_tables_to_cluster(MemoryContext cluster_context);
static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
/*---------------------------------------------------------------------------
@@ -188,16 +188,26 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
/* close relation, keep lock till commit */
table_close(rel, ShareUpdateExclusiveLock);
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
{
/* Do the job. */
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_CLUSTER);
cluster_rel(tableOid, indexOid, ¶ms);
+ pgstat_progress_end_command();
}
else
{
List *rvs;
MemoryContext cluster_context;
+ int64 progress_val[2];
+ const int progress_index[2] = {
+ PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+ };
+
/* Refuse to hold strong locks in a user transaction */
PreventInTransactionBlock(isTopLevel, "CLUSTER");
@@ -206,7 +216,15 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
ALLOCSET_DEFAULT_SIZES);
rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
- cluster_multiple_rels(rvs, params.options);
+
+ /* Report command and number of partitions */
+ progress_val[0] = PROGRESS_CLUSTER_COMMAND_CLUSTER;
+ /* XXX: rvs includes the parent, which is not a "partition" */
+ progress_val[1] = list_length(rvs);
+
+ pgstat_progress_update_multi_param(2, progress_index, progress_val);
+ cluster_multiple_rels(rvs, params.options, true);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -245,7 +263,11 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
* cluster_context.
*/
rvs = get_tables_to_cluster(cluster_context);
- cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_CLUSTER);
+ cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -282,14 +304,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* Check for user-requested abort. */
CHECK_FOR_INTERRUPTS();
- pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
- if (OidIsValid(indexOid))
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_CLUSTER);
- else
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
/*
* We grab exclusive access to the target rel and index for the duration
* of the transaction. (This is redundant for the single-transaction
@@ -300,10 +314,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* If the table has gone away, we can skip processing it */
if (!OldHeap)
- {
- pgstat_progress_end_command();
return;
- }
/*
* Since we may open a new transaction for each relation, we have to check
@@ -319,7 +330,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!pg_class_ownercheck(tableOid, GetUserId()))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -334,7 +344,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (RELATION_IS_OTHER_TEMP(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -346,7 +355,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -357,7 +365,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!get_index_isclustered(indexOid))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
}
@@ -416,7 +423,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!RelationIsPopulated(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -424,7 +430,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -440,8 +445,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
rebuild_relation(OldHeap, indexOid, verbose);
/* NB: rebuild_relation does table_close() on OldHeap */
-
- pgstat_progress_end_command();
}
/*
@@ -1632,9 +1635,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
/* Cluster each relation in a separate transaction */
static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
{
ListCell *lc;
+ off_t childnum = 0;
/* Commit to get out of starting transaction */
PopActiveSnapshot();
@@ -1657,6 +1661,11 @@ cluster_multiple_rels(List *rvs, int options)
cluster_rel(rvtc->tableOid, rvtc->indexOid,
&cluster_params);
+ if (ispartitioned)
+ pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+ ++childnum);
+
+
PopActiveSnapshot();
CommitTransactionCommand();
}
diff --git a/src/backend/commands/vacuum.c b/src/backend/commands/vacuum.c
index c064352e23..05d52858b1 100644
--- a/src/backend/commands/vacuum.c
+++ b/src/backend/commands/vacuum.c
@@ -37,6 +37,7 @@
#include "catalog/pg_namespace.h"
#include "commands/cluster.h"
#include "commands/defrem.h"
+#include "commands/progress.h"
#include "commands/vacuum.h"
#include "miscadmin.h"
#include "nodes/makefuncs.h"
@@ -1937,7 +1938,11 @@ vacuum_rel(Oid relid, RangeVar *relation, VacuumParams *params)
cluster_params.options |= CLUOPT_VERBOSE;
/* VACUUM FULL is now a variant of CLUSTER; see cluster.c */
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, relid);
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
cluster_rel(relid, InvalidOid, &cluster_params);
+ pgstat_progress_end_command();
}
else
table_relation_vacuum(onerel, params, vac_strategy);
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
#define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS 5
#define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED 6
#define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT 7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL 17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE 18
/* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
#define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP 1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b1c9b7bdfe..c888415235 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1943,7 +1943,9 @@ pg_stat_progress_cluster| SELECT s.pid,
s.param5 AS heap_tuples_written,
s.param6 AS heap_blks_total,
s.param7 AS heap_blks_scanned,
- s.param8 AS index_rebuild_count
+ s.param8 AS index_rebuild_count,
+ s.param18 AS partitions_total,
+ s.param19 AS partitions_done
FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
LEFT JOIN pg_database d ON ((s.datid = d.oid)));
pg_stat_progress_copy| SELECT s.pid,
--
2.17.0
--fmvA4kSBHQVZhkR6
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v9-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"
^ permalink raw reply [nested|flat] 73+ messages in thread
* [PATCH v8 3/8] f! progress reporting..
@ 2021-02-09 02:45 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 73+ messages in thread
From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)
This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.
This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
src/backend/catalog/system_views.sql | 4 +-
src/backend/commands/cluster.c | 64 +++++++++++++++++-----------
src/include/commands/progress.h | 3 ++
src/test/regress/expected/rules.out | 4 +-
4 files changed, 47 insertions(+), 28 deletions(-)
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fa58afd9d7..9a3cb5f935 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1069,7 +1069,9 @@ CREATE VIEW pg_stat_progress_cluster AS
S.param5 AS heap_tuples_written,
S.param6 AS heap_blks_total,
S.param7 AS heap_blks_scanned,
- S.param8 AS index_rebuild_count
+ S.param8 AS index_rebuild_count,
+ S.param18 AS partitions_total,
+ S.param19 AS partitions_done
FROM pg_stat_get_progress_info('CLUSTER') AS S
LEFT JOIN pg_database D ON S.datid = D.oid;
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 9b6673867c..cb4fc350c6 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
static List *get_tables_to_cluster(MemoryContext cluster_context);
static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
/*---------------------------------------------------------------------------
@@ -188,16 +188,28 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
/* close relation, keep lock till commit */
table_close(rel, ShareUpdateExclusiveLock);
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
{
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ OidIsValid(indexOid) ? PROGRESS_CLUSTER_COMMAND_CLUSTER :
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
+
/* Do the job. */
cluster_rel(tableOid, indexOid, ¶ms);
+ pgstat_progress_end_command();
}
else
{
List *rvs;
MemoryContext cluster_context;
+ int64 progress_val[2];
+ const int progress_index[] = {
+ PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+ };
+
/* Refuse to hold strong locks in a user transaction */
PreventInTransactionBlock(isTopLevel, "CLUSTER");
@@ -206,7 +218,20 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
ALLOCSET_DEFAULT_SIZES);
rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
- cluster_multiple_rels(rvs, params.options);
+
+ /* Report vacuum full or cluster and number of partitions */
+ progress_val[0] = OidIsValid(indexOid) ?
+ PROGRESS_CLUSTER_COMMAND_CLUSTER :
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL;
+ progress_val[1] = list_length(rvs);
+// This is currently showing the total number of *tables*, including the
+// parent, and intermediate partitions, so the column should be renamed, or
+// we should count the number of leaf partitions
+
+ pgstat_progress_update_multi_param(2, progress_index, progress_val);
+
+ cluster_multiple_rels(rvs, params.options, true);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -245,7 +270,7 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
* cluster_context.
*/
rvs = get_tables_to_cluster(cluster_context);
- cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+ cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -282,14 +307,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* Check for user-requested abort. */
CHECK_FOR_INTERRUPTS();
- pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
- if (OidIsValid(indexOid))
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_CLUSTER);
- else
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
/*
* We grab exclusive access to the target rel and index for the duration
* of the transaction. (This is redundant for the single-transaction
@@ -300,10 +317,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* If the table has gone away, we can skip processing it */
if (!OldHeap)
- {
- pgstat_progress_end_command();
return;
- }
/*
* Since we may open a new transaction for each relation, we have to check
@@ -319,7 +333,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!pg_class_ownercheck(tableOid, GetUserId()))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -334,7 +347,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (RELATION_IS_OTHER_TEMP(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -346,7 +358,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -357,7 +368,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!get_index_isclustered(indexOid))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
}
@@ -416,7 +426,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!RelationIsPopulated(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -424,7 +433,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -440,8 +448,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
rebuild_relation(OldHeap, indexOid, verbose);
/* NB: rebuild_relation does table_close() on OldHeap */
-
- pgstat_progress_end_command();
}
/*
@@ -1366,7 +1372,7 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap,
ReindexParams reindex_params = {0};
int i;
- /* Report that we are now swapping relation files */
+ /* Report that we are now swapping relation files XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_SWAP_REL_FILES);
@@ -1417,13 +1423,13 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap,
else if (newrelpersistence == RELPERSISTENCE_PERMANENT)
reindex_flags |= REINDEX_REL_FORCE_INDEXES_PERMANENT;
- /* Report that we are now reindexing relations */
+ /* Report that we are now reindexing relations XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_REBUILD_INDEX);
reindex_relation(OIDOldHeap, reindex_flags, &reindex_params);
- /* Report that we are now doing clean up */
+ /* Report that we are now doing clean up XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_FINAL_CLEANUP);
@@ -1633,9 +1639,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
/* Cluster each relation in a separate transaction */
static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
{
ListCell *lc;
+ off_t childnum = 0;
/* Commit to get out of starting transaction */
PopActiveSnapshot();
@@ -1658,6 +1665,11 @@ cluster_multiple_rels(List *rvs, int options)
cluster_rel(rvtc->tableOid, rvtc->indexOid,
&cluster_params);
+ if (ispartitioned)
+ pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+ childnum++);
+
+
PopActiveSnapshot();
CommitTransactionCommand();
}
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
#define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS 5
#define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED 6
#define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT 7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL 17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE 18
/* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
#define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP 1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b632d9f2ea..d1fea532bc 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1941,7 +1941,9 @@ pg_stat_progress_cluster| SELECT s.pid,
s.param5 AS heap_tuples_written,
s.param6 AS heap_blks_total,
s.param7 AS heap_blks_scanned,
- s.param8 AS index_rebuild_count
+ s.param8 AS index_rebuild_count,
+ s.param18 AS partitions_total,
+ s.param19 AS partitions_done
FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
LEFT JOIN pg_database d ON ((s.datid = d.oid)));
pg_stat_progress_copy| SELECT s.pid,
--
2.17.0
--O5XBE6gyVG5Rl6Rj
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v8-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"
^ permalink raw reply [nested|flat] 73+ messages in thread
* [PATCH v9 3/8] f! progress reporting..
@ 2021-02-09 02:45 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 73+ messages in thread
From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)
This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.
This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
doc/src/sgml/monitoring.sgml | 21 +++++++++++
src/backend/catalog/system_views.sql | 4 +-
src/backend/commands/cluster.c | 55 ++++++++++++++++------------
src/backend/commands/vacuum.c | 5 +++
src/include/commands/progress.h | 3 ++
src/test/regress/expected/rules.out | 4 +-
6 files changed, 67 insertions(+), 25 deletions(-)
diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml
index 3513e127b7..dca47f6a48 100644
--- a/doc/src/sgml/monitoring.sgml
+++ b/doc/src/sgml/monitoring.sgml
@@ -6231,6 +6231,27 @@ SELECT pg_stat_get_backend_pid(s.backendid) AS pid,
is <literal>rebuilding index</literal>.
</para></entry>
</row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>total_partitions</structfield> <type>bigint</type>
+ </para>
+ When clustering a partitioned table, this column is set to the total
+ number of partitions to be clustered.
+ <para>
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>done_partitions</structfield> <type>bigint</type>
+ </para>
+ <para>
+ When clustering a partitioned table, this column is set to the number
+ of partitions which have been clustered.
+ </para></entry>
+ </row>
+
</tbody>
</tgroup>
</table>
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fc94a73a54..21c27f0437 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1070,7 +1070,9 @@ CREATE VIEW pg_stat_progress_cluster AS
S.param5 AS heap_tuples_written,
S.param6 AS heap_blks_total,
S.param7 AS heap_blks_scanned,
- S.param8 AS index_rebuild_count
+ S.param8 AS index_rebuild_count,
+ S.param18 AS partitions_total,
+ S.param19 AS partitions_done
FROM pg_stat_get_progress_info('CLUSTER') AS S
LEFT JOIN pg_database D ON S.datid = D.oid;
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 0c08ac56dc..7d8d1d8a69 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
static List *get_tables_to_cluster(MemoryContext cluster_context);
static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
/*---------------------------------------------------------------------------
@@ -188,16 +188,26 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
/* close relation, keep lock till commit */
table_close(rel, ShareUpdateExclusiveLock);
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
{
/* Do the job. */
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_CLUSTER);
cluster_rel(tableOid, indexOid, ¶ms);
+ pgstat_progress_end_command();
}
else
{
List *rvs;
MemoryContext cluster_context;
+ int64 progress_val[2];
+ const int progress_index[2] = {
+ PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+ };
+
/* Refuse to hold strong locks in a user transaction */
PreventInTransactionBlock(isTopLevel, "CLUSTER");
@@ -206,7 +216,15 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
ALLOCSET_DEFAULT_SIZES);
rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
- cluster_multiple_rels(rvs, params.options);
+
+ /* Report command and number of partitions */
+ progress_val[0] = PROGRESS_CLUSTER_COMMAND_CLUSTER;
+ /* XXX: rvs includes the parent, which is not a "partition" */
+ progress_val[1] = list_length(rvs);
+
+ pgstat_progress_update_multi_param(2, progress_index, progress_val);
+ cluster_multiple_rels(rvs, params.options, true);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -245,7 +263,11 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
* cluster_context.
*/
rvs = get_tables_to_cluster(cluster_context);
- cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_CLUSTER);
+ cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -282,14 +304,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* Check for user-requested abort. */
CHECK_FOR_INTERRUPTS();
- pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
- if (OidIsValid(indexOid))
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_CLUSTER);
- else
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
/*
* We grab exclusive access to the target rel and index for the duration
* of the transaction. (This is redundant for the single-transaction
@@ -300,10 +314,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* If the table has gone away, we can skip processing it */
if (!OldHeap)
- {
- pgstat_progress_end_command();
return;
- }
/*
* Since we may open a new transaction for each relation, we have to check
@@ -319,7 +330,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!pg_class_ownercheck(tableOid, GetUserId()))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -334,7 +344,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (RELATION_IS_OTHER_TEMP(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -346,7 +355,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -357,7 +365,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!get_index_isclustered(indexOid))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
}
@@ -416,7 +423,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!RelationIsPopulated(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -424,7 +430,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -440,8 +445,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
rebuild_relation(OldHeap, indexOid, verbose);
/* NB: rebuild_relation does table_close() on OldHeap */
-
- pgstat_progress_end_command();
}
/*
@@ -1632,9 +1635,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
/* Cluster each relation in a separate transaction */
static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
{
ListCell *lc;
+ off_t childnum = 0;
/* Commit to get out of starting transaction */
PopActiveSnapshot();
@@ -1657,6 +1661,11 @@ cluster_multiple_rels(List *rvs, int options)
cluster_rel(rvtc->tableOid, rvtc->indexOid,
&cluster_params);
+ if (ispartitioned)
+ pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+ ++childnum);
+
+
PopActiveSnapshot();
CommitTransactionCommand();
}
diff --git a/src/backend/commands/vacuum.c b/src/backend/commands/vacuum.c
index c064352e23..05d52858b1 100644
--- a/src/backend/commands/vacuum.c
+++ b/src/backend/commands/vacuum.c
@@ -37,6 +37,7 @@
#include "catalog/pg_namespace.h"
#include "commands/cluster.h"
#include "commands/defrem.h"
+#include "commands/progress.h"
#include "commands/vacuum.h"
#include "miscadmin.h"
#include "nodes/makefuncs.h"
@@ -1937,7 +1938,11 @@ vacuum_rel(Oid relid, RangeVar *relation, VacuumParams *params)
cluster_params.options |= CLUOPT_VERBOSE;
/* VACUUM FULL is now a variant of CLUSTER; see cluster.c */
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, relid);
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
cluster_rel(relid, InvalidOid, &cluster_params);
+ pgstat_progress_end_command();
}
else
table_relation_vacuum(onerel, params, vac_strategy);
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
#define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS 5
#define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED 6
#define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT 7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL 17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE 18
/* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
#define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP 1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b1c9b7bdfe..c888415235 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1943,7 +1943,9 @@ pg_stat_progress_cluster| SELECT s.pid,
s.param5 AS heap_tuples_written,
s.param6 AS heap_blks_total,
s.param7 AS heap_blks_scanned,
- s.param8 AS index_rebuild_count
+ s.param8 AS index_rebuild_count,
+ s.param18 AS partitions_total,
+ s.param19 AS partitions_done
FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
LEFT JOIN pg_database d ON ((s.datid = d.oid)));
pg_stat_progress_copy| SELECT s.pid,
--
2.17.0
--fmvA4kSBHQVZhkR6
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v9-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"
^ permalink raw reply [nested|flat] 73+ messages in thread
* [PATCH v8 3/8] f! progress reporting..
@ 2021-02-09 02:45 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 73+ messages in thread
From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)
This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.
This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
src/backend/catalog/system_views.sql | 4 +-
src/backend/commands/cluster.c | 64 +++++++++++++++++-----------
src/include/commands/progress.h | 3 ++
src/test/regress/expected/rules.out | 4 +-
4 files changed, 47 insertions(+), 28 deletions(-)
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fa58afd9d7..9a3cb5f935 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1069,7 +1069,9 @@ CREATE VIEW pg_stat_progress_cluster AS
S.param5 AS heap_tuples_written,
S.param6 AS heap_blks_total,
S.param7 AS heap_blks_scanned,
- S.param8 AS index_rebuild_count
+ S.param8 AS index_rebuild_count,
+ S.param18 AS partitions_total,
+ S.param19 AS partitions_done
FROM pg_stat_get_progress_info('CLUSTER') AS S
LEFT JOIN pg_database D ON S.datid = D.oid;
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 9b6673867c..cb4fc350c6 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
static List *get_tables_to_cluster(MemoryContext cluster_context);
static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
/*---------------------------------------------------------------------------
@@ -188,16 +188,28 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
/* close relation, keep lock till commit */
table_close(rel, ShareUpdateExclusiveLock);
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
{
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ OidIsValid(indexOid) ? PROGRESS_CLUSTER_COMMAND_CLUSTER :
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
+
/* Do the job. */
cluster_rel(tableOid, indexOid, ¶ms);
+ pgstat_progress_end_command();
}
else
{
List *rvs;
MemoryContext cluster_context;
+ int64 progress_val[2];
+ const int progress_index[] = {
+ PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+ };
+
/* Refuse to hold strong locks in a user transaction */
PreventInTransactionBlock(isTopLevel, "CLUSTER");
@@ -206,7 +218,20 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
ALLOCSET_DEFAULT_SIZES);
rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
- cluster_multiple_rels(rvs, params.options);
+
+ /* Report vacuum full or cluster and number of partitions */
+ progress_val[0] = OidIsValid(indexOid) ?
+ PROGRESS_CLUSTER_COMMAND_CLUSTER :
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL;
+ progress_val[1] = list_length(rvs);
+// This is currently showing the total number of *tables*, including the
+// parent, and intermediate partitions, so the column should be renamed, or
+// we should count the number of leaf partitions
+
+ pgstat_progress_update_multi_param(2, progress_index, progress_val);
+
+ cluster_multiple_rels(rvs, params.options, true);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -245,7 +270,7 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
* cluster_context.
*/
rvs = get_tables_to_cluster(cluster_context);
- cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+ cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -282,14 +307,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* Check for user-requested abort. */
CHECK_FOR_INTERRUPTS();
- pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
- if (OidIsValid(indexOid))
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_CLUSTER);
- else
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
/*
* We grab exclusive access to the target rel and index for the duration
* of the transaction. (This is redundant for the single-transaction
@@ -300,10 +317,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* If the table has gone away, we can skip processing it */
if (!OldHeap)
- {
- pgstat_progress_end_command();
return;
- }
/*
* Since we may open a new transaction for each relation, we have to check
@@ -319,7 +333,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!pg_class_ownercheck(tableOid, GetUserId()))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -334,7 +347,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (RELATION_IS_OTHER_TEMP(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -346,7 +358,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -357,7 +368,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!get_index_isclustered(indexOid))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
}
@@ -416,7 +426,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!RelationIsPopulated(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -424,7 +433,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -440,8 +448,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
rebuild_relation(OldHeap, indexOid, verbose);
/* NB: rebuild_relation does table_close() on OldHeap */
-
- pgstat_progress_end_command();
}
/*
@@ -1366,7 +1372,7 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap,
ReindexParams reindex_params = {0};
int i;
- /* Report that we are now swapping relation files */
+ /* Report that we are now swapping relation files XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_SWAP_REL_FILES);
@@ -1417,13 +1423,13 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap,
else if (newrelpersistence == RELPERSISTENCE_PERMANENT)
reindex_flags |= REINDEX_REL_FORCE_INDEXES_PERMANENT;
- /* Report that we are now reindexing relations */
+ /* Report that we are now reindexing relations XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_REBUILD_INDEX);
reindex_relation(OIDOldHeap, reindex_flags, &reindex_params);
- /* Report that we are now doing clean up */
+ /* Report that we are now doing clean up XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_FINAL_CLEANUP);
@@ -1633,9 +1639,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
/* Cluster each relation in a separate transaction */
static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
{
ListCell *lc;
+ off_t childnum = 0;
/* Commit to get out of starting transaction */
PopActiveSnapshot();
@@ -1658,6 +1665,11 @@ cluster_multiple_rels(List *rvs, int options)
cluster_rel(rvtc->tableOid, rvtc->indexOid,
&cluster_params);
+ if (ispartitioned)
+ pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+ childnum++);
+
+
PopActiveSnapshot();
CommitTransactionCommand();
}
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
#define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS 5
#define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED 6
#define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT 7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL 17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE 18
/* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
#define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP 1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b632d9f2ea..d1fea532bc 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1941,7 +1941,9 @@ pg_stat_progress_cluster| SELECT s.pid,
s.param5 AS heap_tuples_written,
s.param6 AS heap_blks_total,
s.param7 AS heap_blks_scanned,
- s.param8 AS index_rebuild_count
+ s.param8 AS index_rebuild_count,
+ s.param18 AS partitions_total,
+ s.param19 AS partitions_done
FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
LEFT JOIN pg_database d ON ((s.datid = d.oid)));
pg_stat_progress_copy| SELECT s.pid,
--
2.17.0
--O5XBE6gyVG5Rl6Rj
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v8-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"
^ permalink raw reply [nested|flat] 73+ messages in thread
* [PATCH v9 3/8] f! progress reporting..
@ 2021-02-09 02:45 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 73+ messages in thread
From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)
This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.
This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
doc/src/sgml/monitoring.sgml | 21 +++++++++++
src/backend/catalog/system_views.sql | 4 +-
src/backend/commands/cluster.c | 55 ++++++++++++++++------------
src/backend/commands/vacuum.c | 5 +++
src/include/commands/progress.h | 3 ++
src/test/regress/expected/rules.out | 4 +-
6 files changed, 67 insertions(+), 25 deletions(-)
diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml
index 3513e127b7..dca47f6a48 100644
--- a/doc/src/sgml/monitoring.sgml
+++ b/doc/src/sgml/monitoring.sgml
@@ -6231,6 +6231,27 @@ SELECT pg_stat_get_backend_pid(s.backendid) AS pid,
is <literal>rebuilding index</literal>.
</para></entry>
</row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>total_partitions</structfield> <type>bigint</type>
+ </para>
+ When clustering a partitioned table, this column is set to the total
+ number of partitions to be clustered.
+ <para>
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>done_partitions</structfield> <type>bigint</type>
+ </para>
+ <para>
+ When clustering a partitioned table, this column is set to the number
+ of partitions which have been clustered.
+ </para></entry>
+ </row>
+
</tbody>
</tgroup>
</table>
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fc94a73a54..21c27f0437 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1070,7 +1070,9 @@ CREATE VIEW pg_stat_progress_cluster AS
S.param5 AS heap_tuples_written,
S.param6 AS heap_blks_total,
S.param7 AS heap_blks_scanned,
- S.param8 AS index_rebuild_count
+ S.param8 AS index_rebuild_count,
+ S.param18 AS partitions_total,
+ S.param19 AS partitions_done
FROM pg_stat_get_progress_info('CLUSTER') AS S
LEFT JOIN pg_database D ON S.datid = D.oid;
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 0c08ac56dc..7d8d1d8a69 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
static List *get_tables_to_cluster(MemoryContext cluster_context);
static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
/*---------------------------------------------------------------------------
@@ -188,16 +188,26 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
/* close relation, keep lock till commit */
table_close(rel, ShareUpdateExclusiveLock);
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
{
/* Do the job. */
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_CLUSTER);
cluster_rel(tableOid, indexOid, ¶ms);
+ pgstat_progress_end_command();
}
else
{
List *rvs;
MemoryContext cluster_context;
+ int64 progress_val[2];
+ const int progress_index[2] = {
+ PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+ };
+
/* Refuse to hold strong locks in a user transaction */
PreventInTransactionBlock(isTopLevel, "CLUSTER");
@@ -206,7 +216,15 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
ALLOCSET_DEFAULT_SIZES);
rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
- cluster_multiple_rels(rvs, params.options);
+
+ /* Report command and number of partitions */
+ progress_val[0] = PROGRESS_CLUSTER_COMMAND_CLUSTER;
+ /* XXX: rvs includes the parent, which is not a "partition" */
+ progress_val[1] = list_length(rvs);
+
+ pgstat_progress_update_multi_param(2, progress_index, progress_val);
+ cluster_multiple_rels(rvs, params.options, true);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -245,7 +263,11 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
* cluster_context.
*/
rvs = get_tables_to_cluster(cluster_context);
- cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_CLUSTER);
+ cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -282,14 +304,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* Check for user-requested abort. */
CHECK_FOR_INTERRUPTS();
- pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
- if (OidIsValid(indexOid))
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_CLUSTER);
- else
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
/*
* We grab exclusive access to the target rel and index for the duration
* of the transaction. (This is redundant for the single-transaction
@@ -300,10 +314,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* If the table has gone away, we can skip processing it */
if (!OldHeap)
- {
- pgstat_progress_end_command();
return;
- }
/*
* Since we may open a new transaction for each relation, we have to check
@@ -319,7 +330,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!pg_class_ownercheck(tableOid, GetUserId()))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -334,7 +344,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (RELATION_IS_OTHER_TEMP(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -346,7 +355,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -357,7 +365,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!get_index_isclustered(indexOid))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
}
@@ -416,7 +423,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!RelationIsPopulated(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -424,7 +430,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -440,8 +445,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
rebuild_relation(OldHeap, indexOid, verbose);
/* NB: rebuild_relation does table_close() on OldHeap */
-
- pgstat_progress_end_command();
}
/*
@@ -1632,9 +1635,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
/* Cluster each relation in a separate transaction */
static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
{
ListCell *lc;
+ off_t childnum = 0;
/* Commit to get out of starting transaction */
PopActiveSnapshot();
@@ -1657,6 +1661,11 @@ cluster_multiple_rels(List *rvs, int options)
cluster_rel(rvtc->tableOid, rvtc->indexOid,
&cluster_params);
+ if (ispartitioned)
+ pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+ ++childnum);
+
+
PopActiveSnapshot();
CommitTransactionCommand();
}
diff --git a/src/backend/commands/vacuum.c b/src/backend/commands/vacuum.c
index c064352e23..05d52858b1 100644
--- a/src/backend/commands/vacuum.c
+++ b/src/backend/commands/vacuum.c
@@ -37,6 +37,7 @@
#include "catalog/pg_namespace.h"
#include "commands/cluster.h"
#include "commands/defrem.h"
+#include "commands/progress.h"
#include "commands/vacuum.h"
#include "miscadmin.h"
#include "nodes/makefuncs.h"
@@ -1937,7 +1938,11 @@ vacuum_rel(Oid relid, RangeVar *relation, VacuumParams *params)
cluster_params.options |= CLUOPT_VERBOSE;
/* VACUUM FULL is now a variant of CLUSTER; see cluster.c */
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, relid);
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
cluster_rel(relid, InvalidOid, &cluster_params);
+ pgstat_progress_end_command();
}
else
table_relation_vacuum(onerel, params, vac_strategy);
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
#define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS 5
#define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED 6
#define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT 7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL 17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE 18
/* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
#define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP 1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b1c9b7bdfe..c888415235 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1943,7 +1943,9 @@ pg_stat_progress_cluster| SELECT s.pid,
s.param5 AS heap_tuples_written,
s.param6 AS heap_blks_total,
s.param7 AS heap_blks_scanned,
- s.param8 AS index_rebuild_count
+ s.param8 AS index_rebuild_count,
+ s.param18 AS partitions_total,
+ s.param19 AS partitions_done
FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
LEFT JOIN pg_database d ON ((s.datid = d.oid)));
pg_stat_progress_copy| SELECT s.pid,
--
2.17.0
--fmvA4kSBHQVZhkR6
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v9-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"
^ permalink raw reply [nested|flat] 73+ messages in thread
* [PATCH v8 3/8] f! progress reporting..
@ 2021-02-09 02:45 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 73+ messages in thread
From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)
This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.
This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
src/backend/catalog/system_views.sql | 4 +-
src/backend/commands/cluster.c | 64 +++++++++++++++++-----------
src/include/commands/progress.h | 3 ++
src/test/regress/expected/rules.out | 4 +-
4 files changed, 47 insertions(+), 28 deletions(-)
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fa58afd9d7..9a3cb5f935 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1069,7 +1069,9 @@ CREATE VIEW pg_stat_progress_cluster AS
S.param5 AS heap_tuples_written,
S.param6 AS heap_blks_total,
S.param7 AS heap_blks_scanned,
- S.param8 AS index_rebuild_count
+ S.param8 AS index_rebuild_count,
+ S.param18 AS partitions_total,
+ S.param19 AS partitions_done
FROM pg_stat_get_progress_info('CLUSTER') AS S
LEFT JOIN pg_database D ON S.datid = D.oid;
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 9b6673867c..cb4fc350c6 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
static List *get_tables_to_cluster(MemoryContext cluster_context);
static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
/*---------------------------------------------------------------------------
@@ -188,16 +188,28 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
/* close relation, keep lock till commit */
table_close(rel, ShareUpdateExclusiveLock);
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
{
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ OidIsValid(indexOid) ? PROGRESS_CLUSTER_COMMAND_CLUSTER :
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
+
/* Do the job. */
cluster_rel(tableOid, indexOid, ¶ms);
+ pgstat_progress_end_command();
}
else
{
List *rvs;
MemoryContext cluster_context;
+ int64 progress_val[2];
+ const int progress_index[] = {
+ PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+ };
+
/* Refuse to hold strong locks in a user transaction */
PreventInTransactionBlock(isTopLevel, "CLUSTER");
@@ -206,7 +218,20 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
ALLOCSET_DEFAULT_SIZES);
rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
- cluster_multiple_rels(rvs, params.options);
+
+ /* Report vacuum full or cluster and number of partitions */
+ progress_val[0] = OidIsValid(indexOid) ?
+ PROGRESS_CLUSTER_COMMAND_CLUSTER :
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL;
+ progress_val[1] = list_length(rvs);
+// This is currently showing the total number of *tables*, including the
+// parent, and intermediate partitions, so the column should be renamed, or
+// we should count the number of leaf partitions
+
+ pgstat_progress_update_multi_param(2, progress_index, progress_val);
+
+ cluster_multiple_rels(rvs, params.options, true);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -245,7 +270,7 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
* cluster_context.
*/
rvs = get_tables_to_cluster(cluster_context);
- cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+ cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -282,14 +307,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* Check for user-requested abort. */
CHECK_FOR_INTERRUPTS();
- pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
- if (OidIsValid(indexOid))
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_CLUSTER);
- else
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
/*
* We grab exclusive access to the target rel and index for the duration
* of the transaction. (This is redundant for the single-transaction
@@ -300,10 +317,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* If the table has gone away, we can skip processing it */
if (!OldHeap)
- {
- pgstat_progress_end_command();
return;
- }
/*
* Since we may open a new transaction for each relation, we have to check
@@ -319,7 +333,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!pg_class_ownercheck(tableOid, GetUserId()))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -334,7 +347,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (RELATION_IS_OTHER_TEMP(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -346,7 +358,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -357,7 +368,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!get_index_isclustered(indexOid))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
}
@@ -416,7 +426,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!RelationIsPopulated(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -424,7 +433,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -440,8 +448,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
rebuild_relation(OldHeap, indexOid, verbose);
/* NB: rebuild_relation does table_close() on OldHeap */
-
- pgstat_progress_end_command();
}
/*
@@ -1366,7 +1372,7 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap,
ReindexParams reindex_params = {0};
int i;
- /* Report that we are now swapping relation files */
+ /* Report that we are now swapping relation files XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_SWAP_REL_FILES);
@@ -1417,13 +1423,13 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap,
else if (newrelpersistence == RELPERSISTENCE_PERMANENT)
reindex_flags |= REINDEX_REL_FORCE_INDEXES_PERMANENT;
- /* Report that we are now reindexing relations */
+ /* Report that we are now reindexing relations XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_REBUILD_INDEX);
reindex_relation(OIDOldHeap, reindex_flags, &reindex_params);
- /* Report that we are now doing clean up */
+ /* Report that we are now doing clean up XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_FINAL_CLEANUP);
@@ -1633,9 +1639,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
/* Cluster each relation in a separate transaction */
static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
{
ListCell *lc;
+ off_t childnum = 0;
/* Commit to get out of starting transaction */
PopActiveSnapshot();
@@ -1658,6 +1665,11 @@ cluster_multiple_rels(List *rvs, int options)
cluster_rel(rvtc->tableOid, rvtc->indexOid,
&cluster_params);
+ if (ispartitioned)
+ pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+ childnum++);
+
+
PopActiveSnapshot();
CommitTransactionCommand();
}
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
#define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS 5
#define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED 6
#define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT 7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL 17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE 18
/* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
#define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP 1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b632d9f2ea..d1fea532bc 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1941,7 +1941,9 @@ pg_stat_progress_cluster| SELECT s.pid,
s.param5 AS heap_tuples_written,
s.param6 AS heap_blks_total,
s.param7 AS heap_blks_scanned,
- s.param8 AS index_rebuild_count
+ s.param8 AS index_rebuild_count,
+ s.param18 AS partitions_total,
+ s.param19 AS partitions_done
FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
LEFT JOIN pg_database d ON ((s.datid = d.oid)));
pg_stat_progress_copy| SELECT s.pid,
--
2.17.0
--O5XBE6gyVG5Rl6Rj
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v8-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"
^ permalink raw reply [nested|flat] 73+ messages in thread
* [PATCH v8 3/8] f! progress reporting..
@ 2021-02-09 02:45 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 73+ messages in thread
From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)
This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.
This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
src/backend/catalog/system_views.sql | 4 +-
src/backend/commands/cluster.c | 64 +++++++++++++++++-----------
src/include/commands/progress.h | 3 ++
src/test/regress/expected/rules.out | 4 +-
4 files changed, 47 insertions(+), 28 deletions(-)
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fa58afd9d7..9a3cb5f935 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1069,7 +1069,9 @@ CREATE VIEW pg_stat_progress_cluster AS
S.param5 AS heap_tuples_written,
S.param6 AS heap_blks_total,
S.param7 AS heap_blks_scanned,
- S.param8 AS index_rebuild_count
+ S.param8 AS index_rebuild_count,
+ S.param18 AS partitions_total,
+ S.param19 AS partitions_done
FROM pg_stat_get_progress_info('CLUSTER') AS S
LEFT JOIN pg_database D ON S.datid = D.oid;
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 9b6673867c..cb4fc350c6 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
static List *get_tables_to_cluster(MemoryContext cluster_context);
static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
/*---------------------------------------------------------------------------
@@ -188,16 +188,28 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
/* close relation, keep lock till commit */
table_close(rel, ShareUpdateExclusiveLock);
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
{
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ OidIsValid(indexOid) ? PROGRESS_CLUSTER_COMMAND_CLUSTER :
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
+
/* Do the job. */
cluster_rel(tableOid, indexOid, ¶ms);
+ pgstat_progress_end_command();
}
else
{
List *rvs;
MemoryContext cluster_context;
+ int64 progress_val[2];
+ const int progress_index[] = {
+ PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+ };
+
/* Refuse to hold strong locks in a user transaction */
PreventInTransactionBlock(isTopLevel, "CLUSTER");
@@ -206,7 +218,20 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
ALLOCSET_DEFAULT_SIZES);
rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
- cluster_multiple_rels(rvs, params.options);
+
+ /* Report vacuum full or cluster and number of partitions */
+ progress_val[0] = OidIsValid(indexOid) ?
+ PROGRESS_CLUSTER_COMMAND_CLUSTER :
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL;
+ progress_val[1] = list_length(rvs);
+// This is currently showing the total number of *tables*, including the
+// parent, and intermediate partitions, so the column should be renamed, or
+// we should count the number of leaf partitions
+
+ pgstat_progress_update_multi_param(2, progress_index, progress_val);
+
+ cluster_multiple_rels(rvs, params.options, true);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -245,7 +270,7 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
* cluster_context.
*/
rvs = get_tables_to_cluster(cluster_context);
- cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+ cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -282,14 +307,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* Check for user-requested abort. */
CHECK_FOR_INTERRUPTS();
- pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
- if (OidIsValid(indexOid))
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_CLUSTER);
- else
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
/*
* We grab exclusive access to the target rel and index for the duration
* of the transaction. (This is redundant for the single-transaction
@@ -300,10 +317,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* If the table has gone away, we can skip processing it */
if (!OldHeap)
- {
- pgstat_progress_end_command();
return;
- }
/*
* Since we may open a new transaction for each relation, we have to check
@@ -319,7 +333,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!pg_class_ownercheck(tableOid, GetUserId()))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -334,7 +347,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (RELATION_IS_OTHER_TEMP(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -346,7 +358,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -357,7 +368,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!get_index_isclustered(indexOid))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
}
@@ -416,7 +426,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!RelationIsPopulated(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -424,7 +433,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -440,8 +448,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
rebuild_relation(OldHeap, indexOid, verbose);
/* NB: rebuild_relation does table_close() on OldHeap */
-
- pgstat_progress_end_command();
}
/*
@@ -1366,7 +1372,7 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap,
ReindexParams reindex_params = {0};
int i;
- /* Report that we are now swapping relation files */
+ /* Report that we are now swapping relation files XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_SWAP_REL_FILES);
@@ -1417,13 +1423,13 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap,
else if (newrelpersistence == RELPERSISTENCE_PERMANENT)
reindex_flags |= REINDEX_REL_FORCE_INDEXES_PERMANENT;
- /* Report that we are now reindexing relations */
+ /* Report that we are now reindexing relations XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_REBUILD_INDEX);
reindex_relation(OIDOldHeap, reindex_flags, &reindex_params);
- /* Report that we are now doing clean up */
+ /* Report that we are now doing clean up XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_FINAL_CLEANUP);
@@ -1633,9 +1639,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
/* Cluster each relation in a separate transaction */
static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
{
ListCell *lc;
+ off_t childnum = 0;
/* Commit to get out of starting transaction */
PopActiveSnapshot();
@@ -1658,6 +1665,11 @@ cluster_multiple_rels(List *rvs, int options)
cluster_rel(rvtc->tableOid, rvtc->indexOid,
&cluster_params);
+ if (ispartitioned)
+ pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+ childnum++);
+
+
PopActiveSnapshot();
CommitTransactionCommand();
}
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
#define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS 5
#define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED 6
#define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT 7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL 17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE 18
/* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
#define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP 1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b632d9f2ea..d1fea532bc 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1941,7 +1941,9 @@ pg_stat_progress_cluster| SELECT s.pid,
s.param5 AS heap_tuples_written,
s.param6 AS heap_blks_total,
s.param7 AS heap_blks_scanned,
- s.param8 AS index_rebuild_count
+ s.param8 AS index_rebuild_count,
+ s.param18 AS partitions_total,
+ s.param19 AS partitions_done
FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
LEFT JOIN pg_database d ON ((s.datid = d.oid)));
pg_stat_progress_copy| SELECT s.pid,
--
2.17.0
--O5XBE6gyVG5Rl6Rj
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v8-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"
^ permalink raw reply [nested|flat] 73+ messages in thread
* [PATCH v8 3/8] f! progress reporting..
@ 2021-02-09 02:45 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 73+ messages in thread
From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)
This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.
This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
src/backend/catalog/system_views.sql | 4 +-
src/backend/commands/cluster.c | 64 +++++++++++++++++-----------
src/include/commands/progress.h | 3 ++
src/test/regress/expected/rules.out | 4 +-
4 files changed, 47 insertions(+), 28 deletions(-)
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fa58afd9d7..9a3cb5f935 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1069,7 +1069,9 @@ CREATE VIEW pg_stat_progress_cluster AS
S.param5 AS heap_tuples_written,
S.param6 AS heap_blks_total,
S.param7 AS heap_blks_scanned,
- S.param8 AS index_rebuild_count
+ S.param8 AS index_rebuild_count,
+ S.param18 AS partitions_total,
+ S.param19 AS partitions_done
FROM pg_stat_get_progress_info('CLUSTER') AS S
LEFT JOIN pg_database D ON S.datid = D.oid;
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 9b6673867c..cb4fc350c6 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
static List *get_tables_to_cluster(MemoryContext cluster_context);
static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
/*---------------------------------------------------------------------------
@@ -188,16 +188,28 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
/* close relation, keep lock till commit */
table_close(rel, ShareUpdateExclusiveLock);
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
{
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ OidIsValid(indexOid) ? PROGRESS_CLUSTER_COMMAND_CLUSTER :
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
+
/* Do the job. */
cluster_rel(tableOid, indexOid, ¶ms);
+ pgstat_progress_end_command();
}
else
{
List *rvs;
MemoryContext cluster_context;
+ int64 progress_val[2];
+ const int progress_index[] = {
+ PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+ };
+
/* Refuse to hold strong locks in a user transaction */
PreventInTransactionBlock(isTopLevel, "CLUSTER");
@@ -206,7 +218,20 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
ALLOCSET_DEFAULT_SIZES);
rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
- cluster_multiple_rels(rvs, params.options);
+
+ /* Report vacuum full or cluster and number of partitions */
+ progress_val[0] = OidIsValid(indexOid) ?
+ PROGRESS_CLUSTER_COMMAND_CLUSTER :
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL;
+ progress_val[1] = list_length(rvs);
+// This is currently showing the total number of *tables*, including the
+// parent, and intermediate partitions, so the column should be renamed, or
+// we should count the number of leaf partitions
+
+ pgstat_progress_update_multi_param(2, progress_index, progress_val);
+
+ cluster_multiple_rels(rvs, params.options, true);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -245,7 +270,7 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
* cluster_context.
*/
rvs = get_tables_to_cluster(cluster_context);
- cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+ cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -282,14 +307,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* Check for user-requested abort. */
CHECK_FOR_INTERRUPTS();
- pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
- if (OidIsValid(indexOid))
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_CLUSTER);
- else
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
/*
* We grab exclusive access to the target rel and index for the duration
* of the transaction. (This is redundant for the single-transaction
@@ -300,10 +317,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* If the table has gone away, we can skip processing it */
if (!OldHeap)
- {
- pgstat_progress_end_command();
return;
- }
/*
* Since we may open a new transaction for each relation, we have to check
@@ -319,7 +333,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!pg_class_ownercheck(tableOid, GetUserId()))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -334,7 +347,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (RELATION_IS_OTHER_TEMP(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -346,7 +358,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -357,7 +368,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!get_index_isclustered(indexOid))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
}
@@ -416,7 +426,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!RelationIsPopulated(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -424,7 +433,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -440,8 +448,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
rebuild_relation(OldHeap, indexOid, verbose);
/* NB: rebuild_relation does table_close() on OldHeap */
-
- pgstat_progress_end_command();
}
/*
@@ -1366,7 +1372,7 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap,
ReindexParams reindex_params = {0};
int i;
- /* Report that we are now swapping relation files */
+ /* Report that we are now swapping relation files XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_SWAP_REL_FILES);
@@ -1417,13 +1423,13 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap,
else if (newrelpersistence == RELPERSISTENCE_PERMANENT)
reindex_flags |= REINDEX_REL_FORCE_INDEXES_PERMANENT;
- /* Report that we are now reindexing relations */
+ /* Report that we are now reindexing relations XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_REBUILD_INDEX);
reindex_relation(OIDOldHeap, reindex_flags, &reindex_params);
- /* Report that we are now doing clean up */
+ /* Report that we are now doing clean up XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_FINAL_CLEANUP);
@@ -1633,9 +1639,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
/* Cluster each relation in a separate transaction */
static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
{
ListCell *lc;
+ off_t childnum = 0;
/* Commit to get out of starting transaction */
PopActiveSnapshot();
@@ -1658,6 +1665,11 @@ cluster_multiple_rels(List *rvs, int options)
cluster_rel(rvtc->tableOid, rvtc->indexOid,
&cluster_params);
+ if (ispartitioned)
+ pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+ childnum++);
+
+
PopActiveSnapshot();
CommitTransactionCommand();
}
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
#define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS 5
#define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED 6
#define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT 7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL 17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE 18
/* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
#define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP 1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b632d9f2ea..d1fea532bc 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1941,7 +1941,9 @@ pg_stat_progress_cluster| SELECT s.pid,
s.param5 AS heap_tuples_written,
s.param6 AS heap_blks_total,
s.param7 AS heap_blks_scanned,
- s.param8 AS index_rebuild_count
+ s.param8 AS index_rebuild_count,
+ s.param18 AS partitions_total,
+ s.param19 AS partitions_done
FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
LEFT JOIN pg_database d ON ((s.datid = d.oid)));
pg_stat_progress_copy| SELECT s.pid,
--
2.17.0
--O5XBE6gyVG5Rl6Rj
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v8-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"
^ permalink raw reply [nested|flat] 73+ messages in thread
* [PATCH v9 3/8] f! progress reporting..
@ 2021-02-09 02:45 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 73+ messages in thread
From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)
This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.
This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
doc/src/sgml/monitoring.sgml | 21 +++++++++++
src/backend/catalog/system_views.sql | 4 +-
src/backend/commands/cluster.c | 55 ++++++++++++++++------------
src/backend/commands/vacuum.c | 5 +++
src/include/commands/progress.h | 3 ++
src/test/regress/expected/rules.out | 4 +-
6 files changed, 67 insertions(+), 25 deletions(-)
diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml
index 3513e127b7..dca47f6a48 100644
--- a/doc/src/sgml/monitoring.sgml
+++ b/doc/src/sgml/monitoring.sgml
@@ -6231,6 +6231,27 @@ SELECT pg_stat_get_backend_pid(s.backendid) AS pid,
is <literal>rebuilding index</literal>.
</para></entry>
</row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>total_partitions</structfield> <type>bigint</type>
+ </para>
+ When clustering a partitioned table, this column is set to the total
+ number of partitions to be clustered.
+ <para>
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>done_partitions</structfield> <type>bigint</type>
+ </para>
+ <para>
+ When clustering a partitioned table, this column is set to the number
+ of partitions which have been clustered.
+ </para></entry>
+ </row>
+
</tbody>
</tgroup>
</table>
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fc94a73a54..21c27f0437 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1070,7 +1070,9 @@ CREATE VIEW pg_stat_progress_cluster AS
S.param5 AS heap_tuples_written,
S.param6 AS heap_blks_total,
S.param7 AS heap_blks_scanned,
- S.param8 AS index_rebuild_count
+ S.param8 AS index_rebuild_count,
+ S.param18 AS partitions_total,
+ S.param19 AS partitions_done
FROM pg_stat_get_progress_info('CLUSTER') AS S
LEFT JOIN pg_database D ON S.datid = D.oid;
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 0c08ac56dc..7d8d1d8a69 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
static List *get_tables_to_cluster(MemoryContext cluster_context);
static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
/*---------------------------------------------------------------------------
@@ -188,16 +188,26 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
/* close relation, keep lock till commit */
table_close(rel, ShareUpdateExclusiveLock);
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
{
/* Do the job. */
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_CLUSTER);
cluster_rel(tableOid, indexOid, ¶ms);
+ pgstat_progress_end_command();
}
else
{
List *rvs;
MemoryContext cluster_context;
+ int64 progress_val[2];
+ const int progress_index[2] = {
+ PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+ };
+
/* Refuse to hold strong locks in a user transaction */
PreventInTransactionBlock(isTopLevel, "CLUSTER");
@@ -206,7 +216,15 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
ALLOCSET_DEFAULT_SIZES);
rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
- cluster_multiple_rels(rvs, params.options);
+
+ /* Report command and number of partitions */
+ progress_val[0] = PROGRESS_CLUSTER_COMMAND_CLUSTER;
+ /* XXX: rvs includes the parent, which is not a "partition" */
+ progress_val[1] = list_length(rvs);
+
+ pgstat_progress_update_multi_param(2, progress_index, progress_val);
+ cluster_multiple_rels(rvs, params.options, true);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -245,7 +263,11 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
* cluster_context.
*/
rvs = get_tables_to_cluster(cluster_context);
- cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_CLUSTER);
+ cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -282,14 +304,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* Check for user-requested abort. */
CHECK_FOR_INTERRUPTS();
- pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
- if (OidIsValid(indexOid))
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_CLUSTER);
- else
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
/*
* We grab exclusive access to the target rel and index for the duration
* of the transaction. (This is redundant for the single-transaction
@@ -300,10 +314,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* If the table has gone away, we can skip processing it */
if (!OldHeap)
- {
- pgstat_progress_end_command();
return;
- }
/*
* Since we may open a new transaction for each relation, we have to check
@@ -319,7 +330,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!pg_class_ownercheck(tableOid, GetUserId()))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -334,7 +344,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (RELATION_IS_OTHER_TEMP(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -346,7 +355,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -357,7 +365,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!get_index_isclustered(indexOid))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
}
@@ -416,7 +423,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!RelationIsPopulated(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -424,7 +430,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -440,8 +445,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
rebuild_relation(OldHeap, indexOid, verbose);
/* NB: rebuild_relation does table_close() on OldHeap */
-
- pgstat_progress_end_command();
}
/*
@@ -1632,9 +1635,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
/* Cluster each relation in a separate transaction */
static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
{
ListCell *lc;
+ off_t childnum = 0;
/* Commit to get out of starting transaction */
PopActiveSnapshot();
@@ -1657,6 +1661,11 @@ cluster_multiple_rels(List *rvs, int options)
cluster_rel(rvtc->tableOid, rvtc->indexOid,
&cluster_params);
+ if (ispartitioned)
+ pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+ ++childnum);
+
+
PopActiveSnapshot();
CommitTransactionCommand();
}
diff --git a/src/backend/commands/vacuum.c b/src/backend/commands/vacuum.c
index c064352e23..05d52858b1 100644
--- a/src/backend/commands/vacuum.c
+++ b/src/backend/commands/vacuum.c
@@ -37,6 +37,7 @@
#include "catalog/pg_namespace.h"
#include "commands/cluster.h"
#include "commands/defrem.h"
+#include "commands/progress.h"
#include "commands/vacuum.h"
#include "miscadmin.h"
#include "nodes/makefuncs.h"
@@ -1937,7 +1938,11 @@ vacuum_rel(Oid relid, RangeVar *relation, VacuumParams *params)
cluster_params.options |= CLUOPT_VERBOSE;
/* VACUUM FULL is now a variant of CLUSTER; see cluster.c */
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, relid);
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
cluster_rel(relid, InvalidOid, &cluster_params);
+ pgstat_progress_end_command();
}
else
table_relation_vacuum(onerel, params, vac_strategy);
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
#define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS 5
#define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED 6
#define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT 7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL 17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE 18
/* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
#define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP 1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b1c9b7bdfe..c888415235 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1943,7 +1943,9 @@ pg_stat_progress_cluster| SELECT s.pid,
s.param5 AS heap_tuples_written,
s.param6 AS heap_blks_total,
s.param7 AS heap_blks_scanned,
- s.param8 AS index_rebuild_count
+ s.param8 AS index_rebuild_count,
+ s.param18 AS partitions_total,
+ s.param19 AS partitions_done
FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
LEFT JOIN pg_database d ON ((s.datid = d.oid)));
pg_stat_progress_copy| SELECT s.pid,
--
2.17.0
--fmvA4kSBHQVZhkR6
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v9-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"
^ permalink raw reply [nested|flat] 73+ messages in thread
* [PATCH v8 3/8] f! progress reporting..
@ 2021-02-09 02:45 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 73+ messages in thread
From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)
This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.
This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
src/backend/catalog/system_views.sql | 4 +-
src/backend/commands/cluster.c | 64 +++++++++++++++++-----------
src/include/commands/progress.h | 3 ++
src/test/regress/expected/rules.out | 4 +-
4 files changed, 47 insertions(+), 28 deletions(-)
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fa58afd9d7..9a3cb5f935 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1069,7 +1069,9 @@ CREATE VIEW pg_stat_progress_cluster AS
S.param5 AS heap_tuples_written,
S.param6 AS heap_blks_total,
S.param7 AS heap_blks_scanned,
- S.param8 AS index_rebuild_count
+ S.param8 AS index_rebuild_count,
+ S.param18 AS partitions_total,
+ S.param19 AS partitions_done
FROM pg_stat_get_progress_info('CLUSTER') AS S
LEFT JOIN pg_database D ON S.datid = D.oid;
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 9b6673867c..cb4fc350c6 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
static List *get_tables_to_cluster(MemoryContext cluster_context);
static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
/*---------------------------------------------------------------------------
@@ -188,16 +188,28 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
/* close relation, keep lock till commit */
table_close(rel, ShareUpdateExclusiveLock);
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
{
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ OidIsValid(indexOid) ? PROGRESS_CLUSTER_COMMAND_CLUSTER :
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
+
/* Do the job. */
cluster_rel(tableOid, indexOid, ¶ms);
+ pgstat_progress_end_command();
}
else
{
List *rvs;
MemoryContext cluster_context;
+ int64 progress_val[2];
+ const int progress_index[] = {
+ PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+ };
+
/* Refuse to hold strong locks in a user transaction */
PreventInTransactionBlock(isTopLevel, "CLUSTER");
@@ -206,7 +218,20 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
ALLOCSET_DEFAULT_SIZES);
rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
- cluster_multiple_rels(rvs, params.options);
+
+ /* Report vacuum full or cluster and number of partitions */
+ progress_val[0] = OidIsValid(indexOid) ?
+ PROGRESS_CLUSTER_COMMAND_CLUSTER :
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL;
+ progress_val[1] = list_length(rvs);
+// This is currently showing the total number of *tables*, including the
+// parent, and intermediate partitions, so the column should be renamed, or
+// we should count the number of leaf partitions
+
+ pgstat_progress_update_multi_param(2, progress_index, progress_val);
+
+ cluster_multiple_rels(rvs, params.options, true);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -245,7 +270,7 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
* cluster_context.
*/
rvs = get_tables_to_cluster(cluster_context);
- cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+ cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -282,14 +307,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* Check for user-requested abort. */
CHECK_FOR_INTERRUPTS();
- pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
- if (OidIsValid(indexOid))
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_CLUSTER);
- else
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
/*
* We grab exclusive access to the target rel and index for the duration
* of the transaction. (This is redundant for the single-transaction
@@ -300,10 +317,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* If the table has gone away, we can skip processing it */
if (!OldHeap)
- {
- pgstat_progress_end_command();
return;
- }
/*
* Since we may open a new transaction for each relation, we have to check
@@ -319,7 +333,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!pg_class_ownercheck(tableOid, GetUserId()))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -334,7 +347,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (RELATION_IS_OTHER_TEMP(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -346,7 +358,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -357,7 +368,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!get_index_isclustered(indexOid))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
}
@@ -416,7 +426,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!RelationIsPopulated(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -424,7 +433,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -440,8 +448,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
rebuild_relation(OldHeap, indexOid, verbose);
/* NB: rebuild_relation does table_close() on OldHeap */
-
- pgstat_progress_end_command();
}
/*
@@ -1366,7 +1372,7 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap,
ReindexParams reindex_params = {0};
int i;
- /* Report that we are now swapping relation files */
+ /* Report that we are now swapping relation files XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_SWAP_REL_FILES);
@@ -1417,13 +1423,13 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap,
else if (newrelpersistence == RELPERSISTENCE_PERMANENT)
reindex_flags |= REINDEX_REL_FORCE_INDEXES_PERMANENT;
- /* Report that we are now reindexing relations */
+ /* Report that we are now reindexing relations XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_REBUILD_INDEX);
reindex_relation(OIDOldHeap, reindex_flags, &reindex_params);
- /* Report that we are now doing clean up */
+ /* Report that we are now doing clean up XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_FINAL_CLEANUP);
@@ -1633,9 +1639,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
/* Cluster each relation in a separate transaction */
static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
{
ListCell *lc;
+ off_t childnum = 0;
/* Commit to get out of starting transaction */
PopActiveSnapshot();
@@ -1658,6 +1665,11 @@ cluster_multiple_rels(List *rvs, int options)
cluster_rel(rvtc->tableOid, rvtc->indexOid,
&cluster_params);
+ if (ispartitioned)
+ pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+ childnum++);
+
+
PopActiveSnapshot();
CommitTransactionCommand();
}
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
#define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS 5
#define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED 6
#define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT 7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL 17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE 18
/* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
#define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP 1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b632d9f2ea..d1fea532bc 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1941,7 +1941,9 @@ pg_stat_progress_cluster| SELECT s.pid,
s.param5 AS heap_tuples_written,
s.param6 AS heap_blks_total,
s.param7 AS heap_blks_scanned,
- s.param8 AS index_rebuild_count
+ s.param8 AS index_rebuild_count,
+ s.param18 AS partitions_total,
+ s.param19 AS partitions_done
FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
LEFT JOIN pg_database d ON ((s.datid = d.oid)));
pg_stat_progress_copy| SELECT s.pid,
--
2.17.0
--O5XBE6gyVG5Rl6Rj
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v8-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"
^ permalink raw reply [nested|flat] 73+ messages in thread
* [PATCH v13 02/18] f! progress reporting
@ 2021-02-15 00:31 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 73+ messages in thread
From: Justin Pryzby @ 2021-02-15 00:31 UTC (permalink / raw)
---
src/backend/commands/indexcmds.c | 33 +++++++-------------------------
1 file changed, 7 insertions(+), 26 deletions(-)
diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 4ac1dacd7d..d6567ec231 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1638,40 +1638,20 @@ reindex_invalid_child_indexes(Oid indexRelationId)
.options = REINDEXOPT_CONCURRENTLY
};
- MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX",
- ALLOCSET_DEFAULT_SIZES);
- MemoryContext oldcontext;
- List *childs = find_inheritance_children(indexRelationId, ShareLock);
- List *partitions = NIL;
-
PreventInTransactionBlock(true, "REINDEX INDEX");
- foreach (lc, childs)
+ foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
{
Oid partoid = lfirst_oid(lc);
- /* XXX: need to retrofit progress reporting into it */
- // pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
- // npart++);
-
- if (get_index_isvalid(partoid) ||
- !RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
- continue;
+ if (!get_index_isvalid(partoid) &&
+ RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
+ ReindexRelationConcurrently(partoid, ¶ms);
- /* Save partition OID */
- oldcontext = MemoryContextSwitchTo(ind_context);
- partitions = lappend_oid(partitions, partoid);
- MemoryContextSwitchTo(oldcontext);
+ pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
+ npart++);
}
- /*
- * Process each partition listed in a separate transaction. Note that
- * this commits and then starts a new transaction immediately.
- * XXX: since this is done in 2*N transactions, it could just as well
- * call ReindexRelationConcurrently directly
- */
- ReindexMultipleInternal(partitions, ¶ms);
-
/*
* CIC needs to mark a partitioned index as VALID, which itself
* requires setting READY, which is unset for CIC (even though
@@ -1680,6 +1660,7 @@ reindex_invalid_child_indexes(Oid indexRelationId)
* partitions.
* See also: validatePartitionedIndex().
*/
+ CommandCounterIncrement();
index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
CommandCounterIncrement();
index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
--
2.17.0
--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13-0003-WIP-Add-SKIPVALID-flag-for-more-integration.patch"
^ permalink raw reply [nested|flat] 73+ messages in thread
* [PATCH v13 02/18] f! progress reporting
@ 2021-02-15 00:31 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 73+ messages in thread
From: Justin Pryzby @ 2021-02-15 00:31 UTC (permalink / raw)
---
src/backend/commands/indexcmds.c | 33 +++++++-------------------------
1 file changed, 7 insertions(+), 26 deletions(-)
diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 4ac1dacd7d..d6567ec231 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1638,40 +1638,20 @@ reindex_invalid_child_indexes(Oid indexRelationId)
.options = REINDEXOPT_CONCURRENTLY
};
- MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX",
- ALLOCSET_DEFAULT_SIZES);
- MemoryContext oldcontext;
- List *childs = find_inheritance_children(indexRelationId, ShareLock);
- List *partitions = NIL;
-
PreventInTransactionBlock(true, "REINDEX INDEX");
- foreach (lc, childs)
+ foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
{
Oid partoid = lfirst_oid(lc);
- /* XXX: need to retrofit progress reporting into it */
- // pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
- // npart++);
-
- if (get_index_isvalid(partoid) ||
- !RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
- continue;
+ if (!get_index_isvalid(partoid) &&
+ RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
+ ReindexRelationConcurrently(partoid, ¶ms);
- /* Save partition OID */
- oldcontext = MemoryContextSwitchTo(ind_context);
- partitions = lappend_oid(partitions, partoid);
- MemoryContextSwitchTo(oldcontext);
+ pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
+ npart++);
}
- /*
- * Process each partition listed in a separate transaction. Note that
- * this commits and then starts a new transaction immediately.
- * XXX: since this is done in 2*N transactions, it could just as well
- * call ReindexRelationConcurrently directly
- */
- ReindexMultipleInternal(partitions, ¶ms);
-
/*
* CIC needs to mark a partitioned index as VALID, which itself
* requires setting READY, which is unset for CIC (even though
@@ -1680,6 +1660,7 @@ reindex_invalid_child_indexes(Oid indexRelationId)
* partitions.
* See also: validatePartitionedIndex().
*/
+ CommandCounterIncrement();
index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
CommandCounterIncrement();
index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
--
2.17.0
--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13-0003-WIP-Add-SKIPVALID-flag-for-more-integration.patch"
^ permalink raw reply [nested|flat] 73+ messages in thread
* [PATCH v13 02/18] f! progress reporting
@ 2021-02-15 00:31 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 73+ messages in thread
From: Justin Pryzby @ 2021-02-15 00:31 UTC (permalink / raw)
---
src/backend/commands/indexcmds.c | 33 +++++++-------------------------
1 file changed, 7 insertions(+), 26 deletions(-)
diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 4ac1dacd7d..d6567ec231 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1638,40 +1638,20 @@ reindex_invalid_child_indexes(Oid indexRelationId)
.options = REINDEXOPT_CONCURRENTLY
};
- MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX",
- ALLOCSET_DEFAULT_SIZES);
- MemoryContext oldcontext;
- List *childs = find_inheritance_children(indexRelationId, ShareLock);
- List *partitions = NIL;
-
PreventInTransactionBlock(true, "REINDEX INDEX");
- foreach (lc, childs)
+ foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
{
Oid partoid = lfirst_oid(lc);
- /* XXX: need to retrofit progress reporting into it */
- // pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
- // npart++);
-
- if (get_index_isvalid(partoid) ||
- !RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
- continue;
+ if (!get_index_isvalid(partoid) &&
+ RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
+ ReindexRelationConcurrently(partoid, ¶ms);
- /* Save partition OID */
- oldcontext = MemoryContextSwitchTo(ind_context);
- partitions = lappend_oid(partitions, partoid);
- MemoryContextSwitchTo(oldcontext);
+ pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
+ npart++);
}
- /*
- * Process each partition listed in a separate transaction. Note that
- * this commits and then starts a new transaction immediately.
- * XXX: since this is done in 2*N transactions, it could just as well
- * call ReindexRelationConcurrently directly
- */
- ReindexMultipleInternal(partitions, ¶ms);
-
/*
* CIC needs to mark a partitioned index as VALID, which itself
* requires setting READY, which is unset for CIC (even though
@@ -1680,6 +1660,7 @@ reindex_invalid_child_indexes(Oid indexRelationId)
* partitions.
* See also: validatePartitionedIndex().
*/
+ CommandCounterIncrement();
index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
CommandCounterIncrement();
index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
--
2.17.0
--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13-0003-WIP-Add-SKIPVALID-flag-for-more-integration.patch"
^ permalink raw reply [nested|flat] 73+ messages in thread
* [PATCH v13 02/18] f! progress reporting
@ 2021-02-15 00:31 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 73+ messages in thread
From: Justin Pryzby @ 2021-02-15 00:31 UTC (permalink / raw)
---
src/backend/commands/indexcmds.c | 33 +++++++-------------------------
1 file changed, 7 insertions(+), 26 deletions(-)
diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 4ac1dacd7d..d6567ec231 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1638,40 +1638,20 @@ reindex_invalid_child_indexes(Oid indexRelationId)
.options = REINDEXOPT_CONCURRENTLY
};
- MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX",
- ALLOCSET_DEFAULT_SIZES);
- MemoryContext oldcontext;
- List *childs = find_inheritance_children(indexRelationId, ShareLock);
- List *partitions = NIL;
-
PreventInTransactionBlock(true, "REINDEX INDEX");
- foreach (lc, childs)
+ foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
{
Oid partoid = lfirst_oid(lc);
- /* XXX: need to retrofit progress reporting into it */
- // pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
- // npart++);
-
- if (get_index_isvalid(partoid) ||
- !RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
- continue;
+ if (!get_index_isvalid(partoid) &&
+ RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
+ ReindexRelationConcurrently(partoid, ¶ms);
- /* Save partition OID */
- oldcontext = MemoryContextSwitchTo(ind_context);
- partitions = lappend_oid(partitions, partoid);
- MemoryContextSwitchTo(oldcontext);
+ pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
+ npart++);
}
- /*
- * Process each partition listed in a separate transaction. Note that
- * this commits and then starts a new transaction immediately.
- * XXX: since this is done in 2*N transactions, it could just as well
- * call ReindexRelationConcurrently directly
- */
- ReindexMultipleInternal(partitions, ¶ms);
-
/*
* CIC needs to mark a partitioned index as VALID, which itself
* requires setting READY, which is unset for CIC (even though
@@ -1680,6 +1660,7 @@ reindex_invalid_child_indexes(Oid indexRelationId)
* partitions.
* See also: validatePartitionedIndex().
*/
+ CommandCounterIncrement();
index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
CommandCounterIncrement();
index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
--
2.17.0
--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13-0003-WIP-Add-SKIPVALID-flag-for-more-integration.patch"
^ permalink raw reply [nested|flat] 73+ messages in thread
* [PATCH 2/5] f! progress reporting
@ 2021-02-15 00:31 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 73+ messages in thread
From: Justin Pryzby @ 2021-02-15 00:31 UTC (permalink / raw)
---
src/backend/commands/indexcmds.c | 33 +++++++-------------------------
1 file changed, 7 insertions(+), 26 deletions(-)
diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 9ab1a66971..8f4eab22eb 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1647,40 +1647,20 @@ reindex_invalid_child_indexes(Oid indexRelationId)
.options = REINDEXOPT_CONCURRENTLY
};
- MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX",
- ALLOCSET_DEFAULT_SIZES);
- MemoryContext oldcontext;
- List *childs = find_inheritance_children(indexRelationId, ShareLock);
- List *partitions = NIL;
-
PreventInTransactionBlock(true, "REINDEX INDEX");
- foreach (lc, childs)
+ foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
{
Oid partoid = lfirst_oid(lc);
- /* XXX: need to retrofit progress reporting into it */
- // pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
- // npart++);
-
- if (get_index_isvalid(partoid) ||
- !RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
- continue;
+ if (!get_index_isvalid(partoid) &&
+ RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
+ ReindexRelationConcurrently(partoid, ¶ms);
- /* Save partition OID */
- oldcontext = MemoryContextSwitchTo(ind_context);
- partitions = lappend_oid(partitions, partoid);
- MemoryContextSwitchTo(oldcontext);
+ pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
+ npart++);
}
- /*
- * Process each partition listed in a separate transaction. Note that
- * this commits and then starts a new transaction immediately.
- * XXX: since this is done in 2*N transactions, it could just as well
- * call ReindexRelationConcurrently directly
- */
- ReindexMultipleInternal(partitions, ¶ms);
-
/*
* CIC needs to mark a partitioned index as VALID, which itself
* requires setting READY, which is unset for CIC (even though
@@ -1689,6 +1669,7 @@ reindex_invalid_child_indexes(Oid indexRelationId)
* partitions.
* See also: validatePartitionedIndex().
*/
+ CommandCounterIncrement();
index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
CommandCounterIncrement();
index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
--
2.17.0
--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="0003-WIP-Add-SKIPVALID-flag-for-more-integration.patch"
^ permalink raw reply [nested|flat] 73+ messages in thread
* [PATCH v13 02/18] f! progress reporting
@ 2021-02-15 00:31 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 73+ messages in thread
From: Justin Pryzby @ 2021-02-15 00:31 UTC (permalink / raw)
---
src/backend/commands/indexcmds.c | 33 +++++++-------------------------
1 file changed, 7 insertions(+), 26 deletions(-)
diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 4ac1dacd7d..d6567ec231 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1638,40 +1638,20 @@ reindex_invalid_child_indexes(Oid indexRelationId)
.options = REINDEXOPT_CONCURRENTLY
};
- MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX",
- ALLOCSET_DEFAULT_SIZES);
- MemoryContext oldcontext;
- List *childs = find_inheritance_children(indexRelationId, ShareLock);
- List *partitions = NIL;
-
PreventInTransactionBlock(true, "REINDEX INDEX");
- foreach (lc, childs)
+ foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
{
Oid partoid = lfirst_oid(lc);
- /* XXX: need to retrofit progress reporting into it */
- // pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
- // npart++);
-
- if (get_index_isvalid(partoid) ||
- !RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
- continue;
+ if (!get_index_isvalid(partoid) &&
+ RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
+ ReindexRelationConcurrently(partoid, ¶ms);
- /* Save partition OID */
- oldcontext = MemoryContextSwitchTo(ind_context);
- partitions = lappend_oid(partitions, partoid);
- MemoryContextSwitchTo(oldcontext);
+ pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
+ npart++);
}
- /*
- * Process each partition listed in a separate transaction. Note that
- * this commits and then starts a new transaction immediately.
- * XXX: since this is done in 2*N transactions, it could just as well
- * call ReindexRelationConcurrently directly
- */
- ReindexMultipleInternal(partitions, ¶ms);
-
/*
* CIC needs to mark a partitioned index as VALID, which itself
* requires setting READY, which is unset for CIC (even though
@@ -1680,6 +1660,7 @@ reindex_invalid_child_indexes(Oid indexRelationId)
* partitions.
* See also: validatePartitionedIndex().
*/
+ CommandCounterIncrement();
index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
CommandCounterIncrement();
index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
--
2.17.0
--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13-0003-WIP-Add-SKIPVALID-flag-for-more-integration.patch"
^ permalink raw reply [nested|flat] 73+ messages in thread
* [PATCH v13 02/18] f! progress reporting
@ 2021-02-15 00:31 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 73+ messages in thread
From: Justin Pryzby @ 2021-02-15 00:31 UTC (permalink / raw)
---
src/backend/commands/indexcmds.c | 33 +++++++-------------------------
1 file changed, 7 insertions(+), 26 deletions(-)
diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 4ac1dacd7d..d6567ec231 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1638,40 +1638,20 @@ reindex_invalid_child_indexes(Oid indexRelationId)
.options = REINDEXOPT_CONCURRENTLY
};
- MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX",
- ALLOCSET_DEFAULT_SIZES);
- MemoryContext oldcontext;
- List *childs = find_inheritance_children(indexRelationId, ShareLock);
- List *partitions = NIL;
-
PreventInTransactionBlock(true, "REINDEX INDEX");
- foreach (lc, childs)
+ foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
{
Oid partoid = lfirst_oid(lc);
- /* XXX: need to retrofit progress reporting into it */
- // pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
- // npart++);
-
- if (get_index_isvalid(partoid) ||
- !RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
- continue;
+ if (!get_index_isvalid(partoid) &&
+ RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
+ ReindexRelationConcurrently(partoid, ¶ms);
- /* Save partition OID */
- oldcontext = MemoryContextSwitchTo(ind_context);
- partitions = lappend_oid(partitions, partoid);
- MemoryContextSwitchTo(oldcontext);
+ pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
+ npart++);
}
- /*
- * Process each partition listed in a separate transaction. Note that
- * this commits and then starts a new transaction immediately.
- * XXX: since this is done in 2*N transactions, it could just as well
- * call ReindexRelationConcurrently directly
- */
- ReindexMultipleInternal(partitions, ¶ms);
-
/*
* CIC needs to mark a partitioned index as VALID, which itself
* requires setting READY, which is unset for CIC (even though
@@ -1680,6 +1660,7 @@ reindex_invalid_child_indexes(Oid indexRelationId)
* partitions.
* See also: validatePartitionedIndex().
*/
+ CommandCounterIncrement();
index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
CommandCounterIncrement();
index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
--
2.17.0
--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13-0003-WIP-Add-SKIPVALID-flag-for-more-integration.patch"
^ permalink raw reply [nested|flat] 73+ messages in thread
* [PATCH v13 02/18] f! progress reporting
@ 2021-02-15 00:31 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 73+ messages in thread
From: Justin Pryzby @ 2021-02-15 00:31 UTC (permalink / raw)
---
src/backend/commands/indexcmds.c | 33 +++++++-------------------------
1 file changed, 7 insertions(+), 26 deletions(-)
diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 4ac1dacd7d..d6567ec231 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1638,40 +1638,20 @@ reindex_invalid_child_indexes(Oid indexRelationId)
.options = REINDEXOPT_CONCURRENTLY
};
- MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX",
- ALLOCSET_DEFAULT_SIZES);
- MemoryContext oldcontext;
- List *childs = find_inheritance_children(indexRelationId, ShareLock);
- List *partitions = NIL;
-
PreventInTransactionBlock(true, "REINDEX INDEX");
- foreach (lc, childs)
+ foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
{
Oid partoid = lfirst_oid(lc);
- /* XXX: need to retrofit progress reporting into it */
- // pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
- // npart++);
-
- if (get_index_isvalid(partoid) ||
- !RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
- continue;
+ if (!get_index_isvalid(partoid) &&
+ RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
+ ReindexRelationConcurrently(partoid, ¶ms);
- /* Save partition OID */
- oldcontext = MemoryContextSwitchTo(ind_context);
- partitions = lappend_oid(partitions, partoid);
- MemoryContextSwitchTo(oldcontext);
+ pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
+ npart++);
}
- /*
- * Process each partition listed in a separate transaction. Note that
- * this commits and then starts a new transaction immediately.
- * XXX: since this is done in 2*N transactions, it could just as well
- * call ReindexRelationConcurrently directly
- */
- ReindexMultipleInternal(partitions, ¶ms);
-
/*
* CIC needs to mark a partitioned index as VALID, which itself
* requires setting READY, which is unset for CIC (even though
@@ -1680,6 +1660,7 @@ reindex_invalid_child_indexes(Oid indexRelationId)
* partitions.
* See also: validatePartitionedIndex().
*/
+ CommandCounterIncrement();
index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
CommandCounterIncrement();
index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
--
2.17.0
--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13-0003-WIP-Add-SKIPVALID-flag-for-more-integration.patch"
^ permalink raw reply [nested|flat] 73+ messages in thread
* [PATCH v13 02/18] f! progress reporting
@ 2021-02-15 00:31 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 73+ messages in thread
From: Justin Pryzby @ 2021-02-15 00:31 UTC (permalink / raw)
---
src/backend/commands/indexcmds.c | 33 +++++++-------------------------
1 file changed, 7 insertions(+), 26 deletions(-)
diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 4ac1dacd7d..d6567ec231 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1638,40 +1638,20 @@ reindex_invalid_child_indexes(Oid indexRelationId)
.options = REINDEXOPT_CONCURRENTLY
};
- MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX",
- ALLOCSET_DEFAULT_SIZES);
- MemoryContext oldcontext;
- List *childs = find_inheritance_children(indexRelationId, ShareLock);
- List *partitions = NIL;
-
PreventInTransactionBlock(true, "REINDEX INDEX");
- foreach (lc, childs)
+ foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
{
Oid partoid = lfirst_oid(lc);
- /* XXX: need to retrofit progress reporting into it */
- // pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
- // npart++);
-
- if (get_index_isvalid(partoid) ||
- !RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
- continue;
+ if (!get_index_isvalid(partoid) &&
+ RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
+ ReindexRelationConcurrently(partoid, ¶ms);
- /* Save partition OID */
- oldcontext = MemoryContextSwitchTo(ind_context);
- partitions = lappend_oid(partitions, partoid);
- MemoryContextSwitchTo(oldcontext);
+ pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
+ npart++);
}
- /*
- * Process each partition listed in a separate transaction. Note that
- * this commits and then starts a new transaction immediately.
- * XXX: since this is done in 2*N transactions, it could just as well
- * call ReindexRelationConcurrently directly
- */
- ReindexMultipleInternal(partitions, ¶ms);
-
/*
* CIC needs to mark a partitioned index as VALID, which itself
* requires setting READY, which is unset for CIC (even though
@@ -1680,6 +1660,7 @@ reindex_invalid_child_indexes(Oid indexRelationId)
* partitions.
* See also: validatePartitionedIndex().
*/
+ CommandCounterIncrement();
index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
CommandCounterIncrement();
index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
--
2.17.0
--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13-0003-WIP-Add-SKIPVALID-flag-for-more-integration.patch"
^ permalink raw reply [nested|flat] 73+ messages in thread
* [PATCH v13 02/18] f! progress reporting
@ 2021-02-15 00:31 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 73+ messages in thread
From: Justin Pryzby @ 2021-02-15 00:31 UTC (permalink / raw)
---
src/backend/commands/indexcmds.c | 33 +++++++-------------------------
1 file changed, 7 insertions(+), 26 deletions(-)
diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 4ac1dacd7d..d6567ec231 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1638,40 +1638,20 @@ reindex_invalid_child_indexes(Oid indexRelationId)
.options = REINDEXOPT_CONCURRENTLY
};
- MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX",
- ALLOCSET_DEFAULT_SIZES);
- MemoryContext oldcontext;
- List *childs = find_inheritance_children(indexRelationId, ShareLock);
- List *partitions = NIL;
-
PreventInTransactionBlock(true, "REINDEX INDEX");
- foreach (lc, childs)
+ foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
{
Oid partoid = lfirst_oid(lc);
- /* XXX: need to retrofit progress reporting into it */
- // pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
- // npart++);
-
- if (get_index_isvalid(partoid) ||
- !RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
- continue;
+ if (!get_index_isvalid(partoid) &&
+ RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
+ ReindexRelationConcurrently(partoid, ¶ms);
- /* Save partition OID */
- oldcontext = MemoryContextSwitchTo(ind_context);
- partitions = lappend_oid(partitions, partoid);
- MemoryContextSwitchTo(oldcontext);
+ pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
+ npart++);
}
- /*
- * Process each partition listed in a separate transaction. Note that
- * this commits and then starts a new transaction immediately.
- * XXX: since this is done in 2*N transactions, it could just as well
- * call ReindexRelationConcurrently directly
- */
- ReindexMultipleInternal(partitions, ¶ms);
-
/*
* CIC needs to mark a partitioned index as VALID, which itself
* requires setting READY, which is unset for CIC (even though
@@ -1680,6 +1660,7 @@ reindex_invalid_child_indexes(Oid indexRelationId)
* partitions.
* See also: validatePartitionedIndex().
*/
+ CommandCounterIncrement();
index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
CommandCounterIncrement();
index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
--
2.17.0
--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13-0003-WIP-Add-SKIPVALID-flag-for-more-integration.patch"
^ permalink raw reply [nested|flat] 73+ messages in thread
* [PATCH v13 02/18] f! progress reporting
@ 2021-02-15 00:31 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 73+ messages in thread
From: Justin Pryzby @ 2021-02-15 00:31 UTC (permalink / raw)
---
src/backend/commands/indexcmds.c | 33 +++++++-------------------------
1 file changed, 7 insertions(+), 26 deletions(-)
diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 4ac1dacd7d..d6567ec231 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1638,40 +1638,20 @@ reindex_invalid_child_indexes(Oid indexRelationId)
.options = REINDEXOPT_CONCURRENTLY
};
- MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX",
- ALLOCSET_DEFAULT_SIZES);
- MemoryContext oldcontext;
- List *childs = find_inheritance_children(indexRelationId, ShareLock);
- List *partitions = NIL;
-
PreventInTransactionBlock(true, "REINDEX INDEX");
- foreach (lc, childs)
+ foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
{
Oid partoid = lfirst_oid(lc);
- /* XXX: need to retrofit progress reporting into it */
- // pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
- // npart++);
-
- if (get_index_isvalid(partoid) ||
- !RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
- continue;
+ if (!get_index_isvalid(partoid) &&
+ RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
+ ReindexRelationConcurrently(partoid, ¶ms);
- /* Save partition OID */
- oldcontext = MemoryContextSwitchTo(ind_context);
- partitions = lappend_oid(partitions, partoid);
- MemoryContextSwitchTo(oldcontext);
+ pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
+ npart++);
}
- /*
- * Process each partition listed in a separate transaction. Note that
- * this commits and then starts a new transaction immediately.
- * XXX: since this is done in 2*N transactions, it could just as well
- * call ReindexRelationConcurrently directly
- */
- ReindexMultipleInternal(partitions, ¶ms);
-
/*
* CIC needs to mark a partitioned index as VALID, which itself
* requires setting READY, which is unset for CIC (even though
@@ -1680,6 +1660,7 @@ reindex_invalid_child_indexes(Oid indexRelationId)
* partitions.
* See also: validatePartitionedIndex().
*/
+ CommandCounterIncrement();
index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
CommandCounterIncrement();
index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
--
2.17.0
--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13-0003-WIP-Add-SKIPVALID-flag-for-more-integration.patch"
^ permalink raw reply [nested|flat] 73+ messages in thread
* [PATCH v13 02/18] f! progress reporting
@ 2021-02-15 00:31 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 73+ messages in thread
From: Justin Pryzby @ 2021-02-15 00:31 UTC (permalink / raw)
---
src/backend/commands/indexcmds.c | 33 +++++++-------------------------
1 file changed, 7 insertions(+), 26 deletions(-)
diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 4ac1dacd7d..d6567ec231 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1638,40 +1638,20 @@ reindex_invalid_child_indexes(Oid indexRelationId)
.options = REINDEXOPT_CONCURRENTLY
};
- MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX",
- ALLOCSET_DEFAULT_SIZES);
- MemoryContext oldcontext;
- List *childs = find_inheritance_children(indexRelationId, ShareLock);
- List *partitions = NIL;
-
PreventInTransactionBlock(true, "REINDEX INDEX");
- foreach (lc, childs)
+ foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
{
Oid partoid = lfirst_oid(lc);
- /* XXX: need to retrofit progress reporting into it */
- // pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
- // npart++);
-
- if (get_index_isvalid(partoid) ||
- !RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
- continue;
+ if (!get_index_isvalid(partoid) &&
+ RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
+ ReindexRelationConcurrently(partoid, ¶ms);
- /* Save partition OID */
- oldcontext = MemoryContextSwitchTo(ind_context);
- partitions = lappend_oid(partitions, partoid);
- MemoryContextSwitchTo(oldcontext);
+ pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
+ npart++);
}
- /*
- * Process each partition listed in a separate transaction. Note that
- * this commits and then starts a new transaction immediately.
- * XXX: since this is done in 2*N transactions, it could just as well
- * call ReindexRelationConcurrently directly
- */
- ReindexMultipleInternal(partitions, ¶ms);
-
/*
* CIC needs to mark a partitioned index as VALID, which itself
* requires setting READY, which is unset for CIC (even though
@@ -1680,6 +1660,7 @@ reindex_invalid_child_indexes(Oid indexRelationId)
* partitions.
* See also: validatePartitionedIndex().
*/
+ CommandCounterIncrement();
index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
CommandCounterIncrement();
index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
--
2.17.0
--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13-0003-WIP-Add-SKIPVALID-flag-for-more-integration.patch"
^ permalink raw reply [nested|flat] 73+ messages in thread
* [PATCH v13 02/18] f! progress reporting
@ 2021-02-15 00:31 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 73+ messages in thread
From: Justin Pryzby @ 2021-02-15 00:31 UTC (permalink / raw)
---
src/backend/commands/indexcmds.c | 33 +++++++-------------------------
1 file changed, 7 insertions(+), 26 deletions(-)
diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 4ac1dacd7d..d6567ec231 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1638,40 +1638,20 @@ reindex_invalid_child_indexes(Oid indexRelationId)
.options = REINDEXOPT_CONCURRENTLY
};
- MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX",
- ALLOCSET_DEFAULT_SIZES);
- MemoryContext oldcontext;
- List *childs = find_inheritance_children(indexRelationId, ShareLock);
- List *partitions = NIL;
-
PreventInTransactionBlock(true, "REINDEX INDEX");
- foreach (lc, childs)
+ foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
{
Oid partoid = lfirst_oid(lc);
- /* XXX: need to retrofit progress reporting into it */
- // pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
- // npart++);
-
- if (get_index_isvalid(partoid) ||
- !RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
- continue;
+ if (!get_index_isvalid(partoid) &&
+ RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
+ ReindexRelationConcurrently(partoid, ¶ms);
- /* Save partition OID */
- oldcontext = MemoryContextSwitchTo(ind_context);
- partitions = lappend_oid(partitions, partoid);
- MemoryContextSwitchTo(oldcontext);
+ pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
+ npart++);
}
- /*
- * Process each partition listed in a separate transaction. Note that
- * this commits and then starts a new transaction immediately.
- * XXX: since this is done in 2*N transactions, it could just as well
- * call ReindexRelationConcurrently directly
- */
- ReindexMultipleInternal(partitions, ¶ms);
-
/*
* CIC needs to mark a partitioned index as VALID, which itself
* requires setting READY, which is unset for CIC (even though
@@ -1680,6 +1660,7 @@ reindex_invalid_child_indexes(Oid indexRelationId)
* partitions.
* See also: validatePartitionedIndex().
*/
+ CommandCounterIncrement();
index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
CommandCounterIncrement();
index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
--
2.17.0
--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13-0003-WIP-Add-SKIPVALID-flag-for-more-integration.patch"
^ permalink raw reply [nested|flat] 73+ messages in thread
* [PATCH v13 02/18] f! progress reporting
@ 2021-02-15 00:31 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 73+ messages in thread
From: Justin Pryzby @ 2021-02-15 00:31 UTC (permalink / raw)
---
src/backend/commands/indexcmds.c | 33 +++++++-------------------------
1 file changed, 7 insertions(+), 26 deletions(-)
diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 4ac1dacd7d..d6567ec231 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1638,40 +1638,20 @@ reindex_invalid_child_indexes(Oid indexRelationId)
.options = REINDEXOPT_CONCURRENTLY
};
- MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX",
- ALLOCSET_DEFAULT_SIZES);
- MemoryContext oldcontext;
- List *childs = find_inheritance_children(indexRelationId, ShareLock);
- List *partitions = NIL;
-
PreventInTransactionBlock(true, "REINDEX INDEX");
- foreach (lc, childs)
+ foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
{
Oid partoid = lfirst_oid(lc);
- /* XXX: need to retrofit progress reporting into it */
- // pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
- // npart++);
-
- if (get_index_isvalid(partoid) ||
- !RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
- continue;
+ if (!get_index_isvalid(partoid) &&
+ RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
+ ReindexRelationConcurrently(partoid, ¶ms);
- /* Save partition OID */
- oldcontext = MemoryContextSwitchTo(ind_context);
- partitions = lappend_oid(partitions, partoid);
- MemoryContextSwitchTo(oldcontext);
+ pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
+ npart++);
}
- /*
- * Process each partition listed in a separate transaction. Note that
- * this commits and then starts a new transaction immediately.
- * XXX: since this is done in 2*N transactions, it could just as well
- * call ReindexRelationConcurrently directly
- */
- ReindexMultipleInternal(partitions, ¶ms);
-
/*
* CIC needs to mark a partitioned index as VALID, which itself
* requires setting READY, which is unset for CIC (even though
@@ -1680,6 +1660,7 @@ reindex_invalid_child_indexes(Oid indexRelationId)
* partitions.
* See also: validatePartitionedIndex().
*/
+ CommandCounterIncrement();
index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
CommandCounterIncrement();
index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
--
2.17.0
--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13-0003-WIP-Add-SKIPVALID-flag-for-more-integration.patch"
^ permalink raw reply [nested|flat] 73+ messages in thread
* [PATCH v13 02/18] f! progress reporting
@ 2021-02-15 00:31 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 73+ messages in thread
From: Justin Pryzby @ 2021-02-15 00:31 UTC (permalink / raw)
---
src/backend/commands/indexcmds.c | 33 +++++++-------------------------
1 file changed, 7 insertions(+), 26 deletions(-)
diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 4ac1dacd7d..d6567ec231 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1638,40 +1638,20 @@ reindex_invalid_child_indexes(Oid indexRelationId)
.options = REINDEXOPT_CONCURRENTLY
};
- MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX",
- ALLOCSET_DEFAULT_SIZES);
- MemoryContext oldcontext;
- List *childs = find_inheritance_children(indexRelationId, ShareLock);
- List *partitions = NIL;
-
PreventInTransactionBlock(true, "REINDEX INDEX");
- foreach (lc, childs)
+ foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
{
Oid partoid = lfirst_oid(lc);
- /* XXX: need to retrofit progress reporting into it */
- // pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
- // npart++);
-
- if (get_index_isvalid(partoid) ||
- !RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
- continue;
+ if (!get_index_isvalid(partoid) &&
+ RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
+ ReindexRelationConcurrently(partoid, ¶ms);
- /* Save partition OID */
- oldcontext = MemoryContextSwitchTo(ind_context);
- partitions = lappend_oid(partitions, partoid);
- MemoryContextSwitchTo(oldcontext);
+ pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
+ npart++);
}
- /*
- * Process each partition listed in a separate transaction. Note that
- * this commits and then starts a new transaction immediately.
- * XXX: since this is done in 2*N transactions, it could just as well
- * call ReindexRelationConcurrently directly
- */
- ReindexMultipleInternal(partitions, ¶ms);
-
/*
* CIC needs to mark a partitioned index as VALID, which itself
* requires setting READY, which is unset for CIC (even though
@@ -1680,6 +1660,7 @@ reindex_invalid_child_indexes(Oid indexRelationId)
* partitions.
* See also: validatePartitionedIndex().
*/
+ CommandCounterIncrement();
index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
CommandCounterIncrement();
index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
--
2.17.0
--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13-0003-WIP-Add-SKIPVALID-flag-for-more-integration.patch"
^ permalink raw reply [nested|flat] 73+ messages in thread
* [PATCH v13 02/18] f! progress reporting
@ 2021-02-15 00:31 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 73+ messages in thread
From: Justin Pryzby @ 2021-02-15 00:31 UTC (permalink / raw)
---
src/backend/commands/indexcmds.c | 33 +++++++-------------------------
1 file changed, 7 insertions(+), 26 deletions(-)
diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 4ac1dacd7d..d6567ec231 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1638,40 +1638,20 @@ reindex_invalid_child_indexes(Oid indexRelationId)
.options = REINDEXOPT_CONCURRENTLY
};
- MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX",
- ALLOCSET_DEFAULT_SIZES);
- MemoryContext oldcontext;
- List *childs = find_inheritance_children(indexRelationId, ShareLock);
- List *partitions = NIL;
-
PreventInTransactionBlock(true, "REINDEX INDEX");
- foreach (lc, childs)
+ foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
{
Oid partoid = lfirst_oid(lc);
- /* XXX: need to retrofit progress reporting into it */
- // pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
- // npart++);
-
- if (get_index_isvalid(partoid) ||
- !RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
- continue;
+ if (!get_index_isvalid(partoid) &&
+ RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
+ ReindexRelationConcurrently(partoid, ¶ms);
- /* Save partition OID */
- oldcontext = MemoryContextSwitchTo(ind_context);
- partitions = lappend_oid(partitions, partoid);
- MemoryContextSwitchTo(oldcontext);
+ pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
+ npart++);
}
- /*
- * Process each partition listed in a separate transaction. Note that
- * this commits and then starts a new transaction immediately.
- * XXX: since this is done in 2*N transactions, it could just as well
- * call ReindexRelationConcurrently directly
- */
- ReindexMultipleInternal(partitions, ¶ms);
-
/*
* CIC needs to mark a partitioned index as VALID, which itself
* requires setting READY, which is unset for CIC (even though
@@ -1680,6 +1660,7 @@ reindex_invalid_child_indexes(Oid indexRelationId)
* partitions.
* See also: validatePartitionedIndex().
*/
+ CommandCounterIncrement();
index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
CommandCounterIncrement();
index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
--
2.17.0
--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13-0003-WIP-Add-SKIPVALID-flag-for-more-integration.patch"
^ permalink raw reply [nested|flat] 73+ messages in thread
* [PATCH v13 02/18] f! progress reporting
@ 2021-02-15 00:31 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 73+ messages in thread
From: Justin Pryzby @ 2021-02-15 00:31 UTC (permalink / raw)
---
src/backend/commands/indexcmds.c | 33 +++++++-------------------------
1 file changed, 7 insertions(+), 26 deletions(-)
diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 4ac1dacd7d..d6567ec231 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1638,40 +1638,20 @@ reindex_invalid_child_indexes(Oid indexRelationId)
.options = REINDEXOPT_CONCURRENTLY
};
- MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX",
- ALLOCSET_DEFAULT_SIZES);
- MemoryContext oldcontext;
- List *childs = find_inheritance_children(indexRelationId, ShareLock);
- List *partitions = NIL;
-
PreventInTransactionBlock(true, "REINDEX INDEX");
- foreach (lc, childs)
+ foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
{
Oid partoid = lfirst_oid(lc);
- /* XXX: need to retrofit progress reporting into it */
- // pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
- // npart++);
-
- if (get_index_isvalid(partoid) ||
- !RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
- continue;
+ if (!get_index_isvalid(partoid) &&
+ RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
+ ReindexRelationConcurrently(partoid, ¶ms);
- /* Save partition OID */
- oldcontext = MemoryContextSwitchTo(ind_context);
- partitions = lappend_oid(partitions, partoid);
- MemoryContextSwitchTo(oldcontext);
+ pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
+ npart++);
}
- /*
- * Process each partition listed in a separate transaction. Note that
- * this commits and then starts a new transaction immediately.
- * XXX: since this is done in 2*N transactions, it could just as well
- * call ReindexRelationConcurrently directly
- */
- ReindexMultipleInternal(partitions, ¶ms);
-
/*
* CIC needs to mark a partitioned index as VALID, which itself
* requires setting READY, which is unset for CIC (even though
@@ -1680,6 +1660,7 @@ reindex_invalid_child_indexes(Oid indexRelationId)
* partitions.
* See also: validatePartitionedIndex().
*/
+ CommandCounterIncrement();
index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
CommandCounterIncrement();
index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
--
2.17.0
--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13-0003-WIP-Add-SKIPVALID-flag-for-more-integration.patch"
^ permalink raw reply [nested|flat] 73+ messages in thread
* [PATCH v13 02/18] f! progress reporting
@ 2021-02-15 00:31 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 73+ messages in thread
From: Justin Pryzby @ 2021-02-15 00:31 UTC (permalink / raw)
---
src/backend/commands/indexcmds.c | 33 +++++++-------------------------
1 file changed, 7 insertions(+), 26 deletions(-)
diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 4ac1dacd7d..d6567ec231 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1638,40 +1638,20 @@ reindex_invalid_child_indexes(Oid indexRelationId)
.options = REINDEXOPT_CONCURRENTLY
};
- MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX",
- ALLOCSET_DEFAULT_SIZES);
- MemoryContext oldcontext;
- List *childs = find_inheritance_children(indexRelationId, ShareLock);
- List *partitions = NIL;
-
PreventInTransactionBlock(true, "REINDEX INDEX");
- foreach (lc, childs)
+ foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
{
Oid partoid = lfirst_oid(lc);
- /* XXX: need to retrofit progress reporting into it */
- // pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
- // npart++);
-
- if (get_index_isvalid(partoid) ||
- !RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
- continue;
+ if (!get_index_isvalid(partoid) &&
+ RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
+ ReindexRelationConcurrently(partoid, ¶ms);
- /* Save partition OID */
- oldcontext = MemoryContextSwitchTo(ind_context);
- partitions = lappend_oid(partitions, partoid);
- MemoryContextSwitchTo(oldcontext);
+ pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
+ npart++);
}
- /*
- * Process each partition listed in a separate transaction. Note that
- * this commits and then starts a new transaction immediately.
- * XXX: since this is done in 2*N transactions, it could just as well
- * call ReindexRelationConcurrently directly
- */
- ReindexMultipleInternal(partitions, ¶ms);
-
/*
* CIC needs to mark a partitioned index as VALID, which itself
* requires setting READY, which is unset for CIC (even though
@@ -1680,6 +1660,7 @@ reindex_invalid_child_indexes(Oid indexRelationId)
* partitions.
* See also: validatePartitionedIndex().
*/
+ CommandCounterIncrement();
index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
CommandCounterIncrement();
index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
--
2.17.0
--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13-0003-WIP-Add-SKIPVALID-flag-for-more-integration.patch"
^ permalink raw reply [nested|flat] 73+ messages in thread
* [PATCH v13 02/18] f! progress reporting
@ 2021-02-15 00:31 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 73+ messages in thread
From: Justin Pryzby @ 2021-02-15 00:31 UTC (permalink / raw)
---
src/backend/commands/indexcmds.c | 33 +++++++-------------------------
1 file changed, 7 insertions(+), 26 deletions(-)
diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 4ac1dacd7d..d6567ec231 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1638,40 +1638,20 @@ reindex_invalid_child_indexes(Oid indexRelationId)
.options = REINDEXOPT_CONCURRENTLY
};
- MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX",
- ALLOCSET_DEFAULT_SIZES);
- MemoryContext oldcontext;
- List *childs = find_inheritance_children(indexRelationId, ShareLock);
- List *partitions = NIL;
-
PreventInTransactionBlock(true, "REINDEX INDEX");
- foreach (lc, childs)
+ foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
{
Oid partoid = lfirst_oid(lc);
- /* XXX: need to retrofit progress reporting into it */
- // pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
- // npart++);
-
- if (get_index_isvalid(partoid) ||
- !RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
- continue;
+ if (!get_index_isvalid(partoid) &&
+ RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
+ ReindexRelationConcurrently(partoid, ¶ms);
- /* Save partition OID */
- oldcontext = MemoryContextSwitchTo(ind_context);
- partitions = lappend_oid(partitions, partoid);
- MemoryContextSwitchTo(oldcontext);
+ pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
+ npart++);
}
- /*
- * Process each partition listed in a separate transaction. Note that
- * this commits and then starts a new transaction immediately.
- * XXX: since this is done in 2*N transactions, it could just as well
- * call ReindexRelationConcurrently directly
- */
- ReindexMultipleInternal(partitions, ¶ms);
-
/*
* CIC needs to mark a partitioned index as VALID, which itself
* requires setting READY, which is unset for CIC (even though
@@ -1680,6 +1660,7 @@ reindex_invalid_child_indexes(Oid indexRelationId)
* partitions.
* See also: validatePartitionedIndex().
*/
+ CommandCounterIncrement();
index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
CommandCounterIncrement();
index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
--
2.17.0
--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13-0003-WIP-Add-SKIPVALID-flag-for-more-integration.patch"
^ permalink raw reply [nested|flat] 73+ messages in thread
* [PATCH v13 02/18] f! progress reporting
@ 2021-02-15 00:31 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 73+ messages in thread
From: Justin Pryzby @ 2021-02-15 00:31 UTC (permalink / raw)
---
src/backend/commands/indexcmds.c | 33 +++++++-------------------------
1 file changed, 7 insertions(+), 26 deletions(-)
diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 4ac1dacd7d..d6567ec231 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1638,40 +1638,20 @@ reindex_invalid_child_indexes(Oid indexRelationId)
.options = REINDEXOPT_CONCURRENTLY
};
- MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX",
- ALLOCSET_DEFAULT_SIZES);
- MemoryContext oldcontext;
- List *childs = find_inheritance_children(indexRelationId, ShareLock);
- List *partitions = NIL;
-
PreventInTransactionBlock(true, "REINDEX INDEX");
- foreach (lc, childs)
+ foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
{
Oid partoid = lfirst_oid(lc);
- /* XXX: need to retrofit progress reporting into it */
- // pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
- // npart++);
-
- if (get_index_isvalid(partoid) ||
- !RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
- continue;
+ if (!get_index_isvalid(partoid) &&
+ RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
+ ReindexRelationConcurrently(partoid, ¶ms);
- /* Save partition OID */
- oldcontext = MemoryContextSwitchTo(ind_context);
- partitions = lappend_oid(partitions, partoid);
- MemoryContextSwitchTo(oldcontext);
+ pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
+ npart++);
}
- /*
- * Process each partition listed in a separate transaction. Note that
- * this commits and then starts a new transaction immediately.
- * XXX: since this is done in 2*N transactions, it could just as well
- * call ReindexRelationConcurrently directly
- */
- ReindexMultipleInternal(partitions, ¶ms);
-
/*
* CIC needs to mark a partitioned index as VALID, which itself
* requires setting READY, which is unset for CIC (even though
@@ -1680,6 +1660,7 @@ reindex_invalid_child_indexes(Oid indexRelationId)
* partitions.
* See also: validatePartitionedIndex().
*/
+ CommandCounterIncrement();
index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
CommandCounterIncrement();
index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
--
2.17.0
--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13-0003-WIP-Add-SKIPVALID-flag-for-more-integration.patch"
^ permalink raw reply [nested|flat] 73+ messages in thread
* [PATCH v13 02/18] f! progress reporting
@ 2021-02-15 00:31 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 73+ messages in thread
From: Justin Pryzby @ 2021-02-15 00:31 UTC (permalink / raw)
---
src/backend/commands/indexcmds.c | 33 +++++++-------------------------
1 file changed, 7 insertions(+), 26 deletions(-)
diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 4ac1dacd7d..d6567ec231 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1638,40 +1638,20 @@ reindex_invalid_child_indexes(Oid indexRelationId)
.options = REINDEXOPT_CONCURRENTLY
};
- MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX",
- ALLOCSET_DEFAULT_SIZES);
- MemoryContext oldcontext;
- List *childs = find_inheritance_children(indexRelationId, ShareLock);
- List *partitions = NIL;
-
PreventInTransactionBlock(true, "REINDEX INDEX");
- foreach (lc, childs)
+ foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
{
Oid partoid = lfirst_oid(lc);
- /* XXX: need to retrofit progress reporting into it */
- // pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
- // npart++);
-
- if (get_index_isvalid(partoid) ||
- !RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
- continue;
+ if (!get_index_isvalid(partoid) &&
+ RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
+ ReindexRelationConcurrently(partoid, ¶ms);
- /* Save partition OID */
- oldcontext = MemoryContextSwitchTo(ind_context);
- partitions = lappend_oid(partitions, partoid);
- MemoryContextSwitchTo(oldcontext);
+ pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
+ npart++);
}
- /*
- * Process each partition listed in a separate transaction. Note that
- * this commits and then starts a new transaction immediately.
- * XXX: since this is done in 2*N transactions, it could just as well
- * call ReindexRelationConcurrently directly
- */
- ReindexMultipleInternal(partitions, ¶ms);
-
/*
* CIC needs to mark a partitioned index as VALID, which itself
* requires setting READY, which is unset for CIC (even though
@@ -1680,6 +1660,7 @@ reindex_invalid_child_indexes(Oid indexRelationId)
* partitions.
* See also: validatePartitionedIndex().
*/
+ CommandCounterIncrement();
index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
CommandCounterIncrement();
index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
--
2.17.0
--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13-0003-WIP-Add-SKIPVALID-flag-for-more-integration.patch"
^ permalink raw reply [nested|flat] 73+ messages in thread
* [PATCH v13 02/18] f! progress reporting
@ 2021-02-15 00:31 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 73+ messages in thread
From: Justin Pryzby @ 2021-02-15 00:31 UTC (permalink / raw)
---
src/backend/commands/indexcmds.c | 33 +++++++-------------------------
1 file changed, 7 insertions(+), 26 deletions(-)
diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 4ac1dacd7d..d6567ec231 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1638,40 +1638,20 @@ reindex_invalid_child_indexes(Oid indexRelationId)
.options = REINDEXOPT_CONCURRENTLY
};
- MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX",
- ALLOCSET_DEFAULT_SIZES);
- MemoryContext oldcontext;
- List *childs = find_inheritance_children(indexRelationId, ShareLock);
- List *partitions = NIL;
-
PreventInTransactionBlock(true, "REINDEX INDEX");
- foreach (lc, childs)
+ foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
{
Oid partoid = lfirst_oid(lc);
- /* XXX: need to retrofit progress reporting into it */
- // pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
- // npart++);
-
- if (get_index_isvalid(partoid) ||
- !RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
- continue;
+ if (!get_index_isvalid(partoid) &&
+ RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
+ ReindexRelationConcurrently(partoid, ¶ms);
- /* Save partition OID */
- oldcontext = MemoryContextSwitchTo(ind_context);
- partitions = lappend_oid(partitions, partoid);
- MemoryContextSwitchTo(oldcontext);
+ pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
+ npart++);
}
- /*
- * Process each partition listed in a separate transaction. Note that
- * this commits and then starts a new transaction immediately.
- * XXX: since this is done in 2*N transactions, it could just as well
- * call ReindexRelationConcurrently directly
- */
- ReindexMultipleInternal(partitions, ¶ms);
-
/*
* CIC needs to mark a partitioned index as VALID, which itself
* requires setting READY, which is unset for CIC (even though
@@ -1680,6 +1660,7 @@ reindex_invalid_child_indexes(Oid indexRelationId)
* partitions.
* See also: validatePartitionedIndex().
*/
+ CommandCounterIncrement();
index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
CommandCounterIncrement();
index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
--
2.17.0
--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13-0003-WIP-Add-SKIPVALID-flag-for-more-integration.patch"
^ permalink raw reply [nested|flat] 73+ messages in thread
* [PATCH v13 02/18] f! progress reporting
@ 2021-02-15 00:31 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 73+ messages in thread
From: Justin Pryzby @ 2021-02-15 00:31 UTC (permalink / raw)
---
src/backend/commands/indexcmds.c | 33 +++++++-------------------------
1 file changed, 7 insertions(+), 26 deletions(-)
diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 4ac1dacd7d..d6567ec231 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1638,40 +1638,20 @@ reindex_invalid_child_indexes(Oid indexRelationId)
.options = REINDEXOPT_CONCURRENTLY
};
- MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX",
- ALLOCSET_DEFAULT_SIZES);
- MemoryContext oldcontext;
- List *childs = find_inheritance_children(indexRelationId, ShareLock);
- List *partitions = NIL;
-
PreventInTransactionBlock(true, "REINDEX INDEX");
- foreach (lc, childs)
+ foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
{
Oid partoid = lfirst_oid(lc);
- /* XXX: need to retrofit progress reporting into it */
- // pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
- // npart++);
-
- if (get_index_isvalid(partoid) ||
- !RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
- continue;
+ if (!get_index_isvalid(partoid) &&
+ RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
+ ReindexRelationConcurrently(partoid, ¶ms);
- /* Save partition OID */
- oldcontext = MemoryContextSwitchTo(ind_context);
- partitions = lappend_oid(partitions, partoid);
- MemoryContextSwitchTo(oldcontext);
+ pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
+ npart++);
}
- /*
- * Process each partition listed in a separate transaction. Note that
- * this commits and then starts a new transaction immediately.
- * XXX: since this is done in 2*N transactions, it could just as well
- * call ReindexRelationConcurrently directly
- */
- ReindexMultipleInternal(partitions, ¶ms);
-
/*
* CIC needs to mark a partitioned index as VALID, which itself
* requires setting READY, which is unset for CIC (even though
@@ -1680,6 +1660,7 @@ reindex_invalid_child_indexes(Oid indexRelationId)
* partitions.
* See also: validatePartitionedIndex().
*/
+ CommandCounterIncrement();
index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
CommandCounterIncrement();
index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
--
2.17.0
--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13-0003-WIP-Add-SKIPVALID-flag-for-more-integration.patch"
^ permalink raw reply [nested|flat] 73+ messages in thread
* Re: post-freeze damage control
@ 2024-04-08 23:33 Michael Paquier <[email protected]>
2024-04-09 11:24 ` Re: post-freeze damage control Tomas Vondra <[email protected]>
2024-04-10 01:49 ` Re: post-freeze damage control John Naylor <[email protected]>
0 siblings, 2 replies; 73+ messages in thread
From: Michael Paquier @ 2024-04-08 23:33 UTC (permalink / raw)
To: Tomas Vondra <[email protected]>; +Cc: Robert Haas <[email protected]>; pgsql-hackers
On Tue, Apr 09, 2024 at 01:16:02AM +0200, Tomas Vondra wrote:
> I don't feel too particularly worried about this. Yes, backups are super
> important because it's often the only thing you have left when things go
> wrong, and the incremental aspect is all new. The code I've seen while
> doing the CoW-related patches seemed very precise and careful, and the
> one bug we found & fixed does not make it bad.
>
> Sure, I can't rule there being more bugs, but I've been doing some
> pretty extensive stress testing of this (doing incremental backups +
> combinebackup, and comparing the results against the source, and that
> sort of stuff). And so far only that single bug this way. I'm still
> doing this randomized stress testing, with more and more complex
> workloads etc. and I'll let keep doing that for a while.
>
> Maybe I'm a bit too happy-go-lucky, but IMO the risk here is limited.
Even if there's a critical bug, there are still other ways to take
backups, so there is an exit route even if a problem is found and even
if this problem requires a complex solution to be able to work
correctly.
This worries me less than other patches like the ones around heap
changes or the radix tree stuff for TID collection plugged into
vacuum, which don't have explicit on/off switches AFAIK.
--
Michael
Attachments:
[application/pgp-signature] signature.asc (833B, ../../[email protected]/2-signature.asc)
download
^ permalink raw reply [nested|flat] 73+ messages in thread
* Re: post-freeze damage control
2024-04-08 23:33 Re: post-freeze damage control Michael Paquier <[email protected]>
@ 2024-04-09 11:24 ` Tomas Vondra <[email protected]>
1 sibling, 0 replies; 73+ messages in thread
From: Tomas Vondra @ 2024-04-09 11:24 UTC (permalink / raw)
To: Michael Paquier <[email protected]>; +Cc: Robert Haas <[email protected]>; pgsql-hackers
On 4/9/24 01:33, Michael Paquier wrote:
> On Tue, Apr 09, 2024 at 01:16:02AM +0200, Tomas Vondra wrote:
>> I don't feel too particularly worried about this. Yes, backups are super
>> important because it's often the only thing you have left when things go
>> wrong, and the incremental aspect is all new. The code I've seen while
>> doing the CoW-related patches seemed very precise and careful, and the
>> one bug we found & fixed does not make it bad.
>>
>> Sure, I can't rule there being more bugs, but I've been doing some
>> pretty extensive stress testing of this (doing incremental backups +
>> combinebackup, and comparing the results against the source, and that
>> sort of stuff). And so far only that single bug this way. I'm still
>> doing this randomized stress testing, with more and more complex
>> workloads etc. and I'll let keep doing that for a while.
>>
>> Maybe I'm a bit too happy-go-lucky, but IMO the risk here is limited.
>
> Even if there's a critical bug, there are still other ways to take
> backups, so there is an exit route even if a problem is found and even
> if this problem requires a complex solution to be able to work
> correctly.
>
I think it's a bit more nuanced, because it's about backups/restore. The
bug might be subtle, and you won't learn about it until the moment when
you need to restore (or perhaps even long after that). At which point
"You might have taken the backup in some other way." is not really a
viable exit route.
Anyway, I'm still not worried about this particular feature, and I'll
keep doing the stress testing.
regards
--
Tomas Vondra
EnterpriseDB: http://www.enterprisedb.com
The Enterprise PostgreSQL Company
^ permalink raw reply [nested|flat] 73+ messages in thread
* Re: post-freeze damage control
2024-04-08 23:33 Re: post-freeze damage control Michael Paquier <[email protected]>
@ 2024-04-10 01:49 ` John Naylor <[email protected]>
1 sibling, 0 replies; 73+ messages in thread
From: John Naylor @ 2024-04-10 01:49 UTC (permalink / raw)
To: Michael Paquier <[email protected]>; +Cc: Tomas Vondra <[email protected]>; Robert Haas <[email protected]>; pgsql-hackers
On Tue, Apr 9, 2024 at 2:47 AM Robert Haas <[email protected]> wrote:
>
> - I'm slightly worried about the TID store work (ee1b30f12, 30e144287,
> 667e65aac35), perhaps for no reason. Actually, the results seem really
> impressive,
First, thanks for the complement. I actually suspect if we had this
years ago, it might never have occurred to anyone to go through the
trouble of adding parallel index cleanup.
In a funny way, it's almost too effective -- the claim is that m_w_m
over 1GB is perfectly usable, but I haven't been able to get anywere
near that via vacuum (we have indirectly, via bespoke dev code,
but...). I don't have easy access to hardware that can hold a table
big enough to do so -- vacuuming 1 billion records stays under 400MB.
> and a very quick look at some of the commits seemed kind
> of reassuring. But, like the changes to pruning and freezing, this is
> making some really fundamental changes to critical code. In this case,
> it's code that has evolved very little over the years and seems to
> have now evolved quite a lot.
True. I'd say that at a high level, storage and retrieval of TIDs is a
lot simpler conceptually than other aspects of vacuuming. The
low-level guts are now much more complex, but I'm confident it won't
just output a wrong answer. That aspect has been working for a long
time, and when it has broken during development, it fails very quickly
and obviously.
The more challenging aspects are less cut-and-dried, like memory
management, delegation of responsibility, how to expose locking (which
vacuum doesn't even use), readability/maintainability. Those are more
subjective, but it seems to have finally clicked into place in a way
that feels like the right trade-offs have been made. That's hand-wavy,
I realize.
The more recent follow-up commits are pieces that were discussed and
planned for earlier, but have had less review and were left out from
the initial commits since they're not essential to the functionality.
I judged that test coverage was enough to have confidence in them.
On Tue, Apr 9, 2024 at 6:33 AM Michael Paquier <[email protected]> wrote:
>
> This worries me less than other patches like the ones around heap
> changes or the radix tree stuff for TID collection plugged into
> vacuum, which don't have explicit on/off switches AFAIK.
Yes, there is no switch. Interestingly enough, the previous item array
ended up with its own escape hatch to tamp down its eager allocation,
autovacuum_work_mem. Echoing what I said to Robert, if we had the new
storage years ago, I doubt this GUC would ever have been proposed.
I'll also mention the array is still in the code base, but only in a
test module as a standard to test against. Hopefully that offers some
reassurance.
^ permalink raw reply [nested|flat] 73+ messages in thread
end of thread, other threads:[~2024-04-10 01:49 UTC | newest]
Thread overview: 73+ messages (download: mbox mbox.gz follow: Atom feed)
-- links below jump to the message on this page --
2021-02-09 02:45 [PATCH v9 3/8] f! progress reporting.. Justin Pryzby <[email protected]>
2021-02-09 02:45 [PATCH v9 3/8] f! progress reporting.. Justin Pryzby <[email protected]>
2021-02-09 02:45 [PATCH v9 3/8] f! progress reporting.. Justin Pryzby <[email protected]>
2021-02-09 02:45 [PATCH v10 3/8] f! progress reporting.. Justin Pryzby <[email protected]>
2021-02-09 02:45 [PATCH v8 3/8] f! progress reporting.. Justin Pryzby <[email protected]>
2021-02-09 02:45 [PATCH v8 3/8] f! progress reporting.. Justin Pryzby <[email protected]>
2021-02-09 02:45 [PATCH v8 3/8] f! progress reporting.. Justin Pryzby <[email protected]>
2021-02-09 02:45 [PATCH v9 3/8] f! progress reporting.. Justin Pryzby <[email protected]>
2021-02-09 02:45 [PATCH v8 3/8] f! progress reporting.. Justin Pryzby <[email protected]>
2021-02-09 02:45 [PATCH v9 3/8] f! progress reporting.. Justin Pryzby <[email protected]>
2021-02-09 02:45 [PATCH v9 3/8] f! progress reporting.. Justin Pryzby <[email protected]>
2021-02-09 02:45 [PATCH v8 3/8] f! progress reporting.. Justin Pryzby <[email protected]>
2021-02-09 02:45 [PATCH v9 3/8] f! progress reporting.. Justin Pryzby <[email protected]>
2021-02-09 02:45 [PATCH v9 3/8] f! progress reporting.. Justin Pryzby <[email protected]>
2021-02-09 02:45 [PATCH v9 3/8] f! progress reporting.. Justin Pryzby <[email protected]>
2021-02-09 02:45 [PATCH v9 3/8] f! progress reporting.. Justin Pryzby <[email protected]>
2021-02-09 02:45 [PATCH v8 3/8] f! progress reporting.. Justin Pryzby <[email protected]>
2021-02-09 02:45 [PATCH v8 3/8] f! progress reporting.. Justin Pryzby <[email protected]>
2021-02-09 02:45 [PATCH v8 3/8] f! progress reporting.. Justin Pryzby <[email protected]>
2021-02-09 02:45 [PATCH v8 3/8] f! progress reporting.. Justin Pryzby <[email protected]>
2021-02-09 02:45 [PATCH v9 3/8] f! progress reporting.. Justin Pryzby <[email protected]>
2021-02-09 02:45 [PATCH v8 3/8] f! progress reporting.. Justin Pryzby <[email protected]>
2021-02-09 02:45 [PATCH v9 3/8] f! progress reporting.. Justin Pryzby <[email protected]>
2021-02-09 02:45 [PATCH v8 3/8] f! progress reporting.. Justin Pryzby <[email protected]>
2021-02-09 02:45 [PATCH v8 3/8] f! progress reporting.. Justin Pryzby <[email protected]>
2021-02-09 02:45 [PATCH v8 3/8] f! progress reporting.. Justin Pryzby <[email protected]>
2021-02-09 02:45 [PATCH v9 3/8] f! progress reporting.. Justin Pryzby <[email protected]>
2021-02-09 02:45 [PATCH v9 3/8] f! progress reporting.. Justin Pryzby <[email protected]>
2021-02-09 02:45 [PATCH v9 3/8] f! progress reporting.. Justin Pryzby <[email protected]>
2021-02-09 02:45 [PATCH v9 3/8] f! progress reporting.. Justin Pryzby <[email protected]>
2021-02-09 02:45 [PATCH v8 3/8] f! progress reporting.. Justin Pryzby <[email protected]>
2021-02-09 02:45 [PATCH v9 3/8] f! progress reporting.. Justin Pryzby <[email protected]>
2021-02-09 02:45 [PATCH v9 3/8] f! progress reporting.. Justin Pryzby <[email protected]>
2021-02-09 02:45 [PATCH v8 3/8] f! progress reporting.. Justin Pryzby <[email protected]>
2021-02-09 02:45 [PATCH v9 3/8] f! progress reporting.. Justin Pryzby <[email protected]>
2021-02-09 02:45 [PATCH v9 3/8] f! progress reporting.. Justin Pryzby <[email protected]>
2021-02-09 02:45 [PATCH v8 3/8] f! progress reporting.. Justin Pryzby <[email protected]>
2021-02-09 02:45 [PATCH v9 3/8] f! progress reporting.. Justin Pryzby <[email protected]>
2021-02-09 02:45 [PATCH v8 3/8] f! progress reporting.. Justin Pryzby <[email protected]>
2021-02-09 02:45 [PATCH v9 3/8] f! progress reporting.. Justin Pryzby <[email protected]>
2021-02-09 02:45 [PATCH v8 3/8] f! progress reporting.. Justin Pryzby <[email protected]>
2021-02-09 02:45 [PATCH v9 3/8] f! progress reporting.. Justin Pryzby <[email protected]>
2021-02-09 02:45 [PATCH v8 3/8] f! progress reporting.. Justin Pryzby <[email protected]>
2021-02-09 02:45 [PATCH v8 3/8] f! progress reporting.. Justin Pryzby <[email protected]>
2021-02-09 02:45 [PATCH v8 3/8] f! progress reporting.. Justin Pryzby <[email protected]>
2021-02-09 02:45 [PATCH v9 3/8] f! progress reporting.. Justin Pryzby <[email protected]>
2021-02-09 02:45 [PATCH v8 3/8] f! progress reporting.. Justin Pryzby <[email protected]>
2021-02-15 00:31 [PATCH v13 02/18] f! progress reporting Justin Pryzby <[email protected]>
2021-02-15 00:31 [PATCH v13 02/18] f! progress reporting Justin Pryzby <[email protected]>
2021-02-15 00:31 [PATCH v13 02/18] f! progress reporting Justin Pryzby <[email protected]>
2021-02-15 00:31 [PATCH v13 02/18] f! progress reporting Justin Pryzby <[email protected]>
2021-02-15 00:31 [PATCH 2/5] f! progress reporting Justin Pryzby <[email protected]>
2021-02-15 00:31 [PATCH v13 02/18] f! progress reporting Justin Pryzby <[email protected]>
2021-02-15 00:31 [PATCH v13 02/18] f! progress reporting Justin Pryzby <[email protected]>
2021-02-15 00:31 [PATCH v13 02/18] f! progress reporting Justin Pryzby <[email protected]>
2021-02-15 00:31 [PATCH v13 02/18] f! progress reporting Justin Pryzby <[email protected]>
2021-02-15 00:31 [PATCH v13 02/18] f! progress reporting Justin Pryzby <[email protected]>
2021-02-15 00:31 [PATCH v13 02/18] f! progress reporting Justin Pryzby <[email protected]>
2021-02-15 00:31 [PATCH v13 02/18] f! progress reporting Justin Pryzby <[email protected]>
2021-02-15 00:31 [PATCH v13 02/18] f! progress reporting Justin Pryzby <[email protected]>
2021-02-15 00:31 [PATCH v13 02/18] f! progress reporting Justin Pryzby <[email protected]>
2021-02-15 00:31 [PATCH v13 02/18] f! progress reporting Justin Pryzby <[email protected]>
2021-02-15 00:31 [PATCH v13 02/18] f! progress reporting Justin Pryzby <[email protected]>
2021-02-15 00:31 [PATCH v13 02/18] f! progress reporting Justin Pryzby <[email protected]>
2021-02-15 00:31 [PATCH v13 02/18] f! progress reporting Justin Pryzby <[email protected]>
2021-02-15 00:31 [PATCH v13 02/18] f! progress reporting Justin Pryzby <[email protected]>
2021-02-15 00:31 [PATCH v13 02/18] f! progress reporting Justin Pryzby <[email protected]>
2021-02-15 00:31 [PATCH v13 02/18] f! progress reporting Justin Pryzby <[email protected]>
2021-02-15 00:31 [PATCH v13 02/18] f! progress reporting Justin Pryzby <[email protected]>
2021-02-15 00:31 [PATCH v13 02/18] f! progress reporting Justin Pryzby <[email protected]>
2024-04-08 23:33 Re: post-freeze damage control Michael Paquier <[email protected]>
2024-04-09 11:24 ` Re: post-freeze damage control Tomas Vondra <[email protected]>
2024-04-10 01:49 ` Re: post-freeze damage control John Naylor <[email protected]>
This inbox is served by agora; see mirroring instructions
for how to clone and mirror all data and code used for this inbox