public inbox for [email protected]
help / color / mirror / Atom feed[PATCH v8 3/8] f! progress reporting..
73+ messages / 4 participants
[nested] [flat]
* [PATCH v9 3/8] f! progress reporting..
@ 2021-02-09 02:45 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 73+ messages in thread
From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)
This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.
This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
doc/src/sgml/monitoring.sgml | 21 +++++++++++
src/backend/catalog/system_views.sql | 4 +-
src/backend/commands/cluster.c | 55 ++++++++++++++++------------
src/backend/commands/vacuum.c | 5 +++
src/include/commands/progress.h | 3 ++
src/test/regress/expected/rules.out | 4 +-
6 files changed, 67 insertions(+), 25 deletions(-)
diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml
index 3513e127b7..dca47f6a48 100644
--- a/doc/src/sgml/monitoring.sgml
+++ b/doc/src/sgml/monitoring.sgml
@@ -6231,6 +6231,27 @@ SELECT pg_stat_get_backend_pid(s.backendid) AS pid,
is <literal>rebuilding index</literal>.
</para></entry>
</row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>total_partitions</structfield> <type>bigint</type>
+ </para>
+ When clustering a partitioned table, this column is set to the total
+ number of partitions to be clustered.
+ <para>
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>done_partitions</structfield> <type>bigint</type>
+ </para>
+ <para>
+ When clustering a partitioned table, this column is set to the number
+ of partitions which have been clustered.
+ </para></entry>
+ </row>
+
</tbody>
</tgroup>
</table>
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fc94a73a54..21c27f0437 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1070,7 +1070,9 @@ CREATE VIEW pg_stat_progress_cluster AS
S.param5 AS heap_tuples_written,
S.param6 AS heap_blks_total,
S.param7 AS heap_blks_scanned,
- S.param8 AS index_rebuild_count
+ S.param8 AS index_rebuild_count,
+ S.param18 AS partitions_total,
+ S.param19 AS partitions_done
FROM pg_stat_get_progress_info('CLUSTER') AS S
LEFT JOIN pg_database D ON S.datid = D.oid;
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 0c08ac56dc..7d8d1d8a69 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
static List *get_tables_to_cluster(MemoryContext cluster_context);
static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
/*---------------------------------------------------------------------------
@@ -188,16 +188,26 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
/* close relation, keep lock till commit */
table_close(rel, ShareUpdateExclusiveLock);
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
{
/* Do the job. */
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_CLUSTER);
cluster_rel(tableOid, indexOid, ¶ms);
+ pgstat_progress_end_command();
}
else
{
List *rvs;
MemoryContext cluster_context;
+ int64 progress_val[2];
+ const int progress_index[2] = {
+ PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+ };
+
/* Refuse to hold strong locks in a user transaction */
PreventInTransactionBlock(isTopLevel, "CLUSTER");
@@ -206,7 +216,15 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
ALLOCSET_DEFAULT_SIZES);
rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
- cluster_multiple_rels(rvs, params.options);
+
+ /* Report command and number of partitions */
+ progress_val[0] = PROGRESS_CLUSTER_COMMAND_CLUSTER;
+ /* XXX: rvs includes the parent, which is not a "partition" */
+ progress_val[1] = list_length(rvs);
+
+ pgstat_progress_update_multi_param(2, progress_index, progress_val);
+ cluster_multiple_rels(rvs, params.options, true);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -245,7 +263,11 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
* cluster_context.
*/
rvs = get_tables_to_cluster(cluster_context);
- cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_CLUSTER);
+ cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -282,14 +304,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* Check for user-requested abort. */
CHECK_FOR_INTERRUPTS();
- pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
- if (OidIsValid(indexOid))
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_CLUSTER);
- else
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
/*
* We grab exclusive access to the target rel and index for the duration
* of the transaction. (This is redundant for the single-transaction
@@ -300,10 +314,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* If the table has gone away, we can skip processing it */
if (!OldHeap)
- {
- pgstat_progress_end_command();
return;
- }
/*
* Since we may open a new transaction for each relation, we have to check
@@ -319,7 +330,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!pg_class_ownercheck(tableOid, GetUserId()))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -334,7 +344,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (RELATION_IS_OTHER_TEMP(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -346,7 +355,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -357,7 +365,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!get_index_isclustered(indexOid))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
}
@@ -416,7 +423,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!RelationIsPopulated(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -424,7 +430,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -440,8 +445,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
rebuild_relation(OldHeap, indexOid, verbose);
/* NB: rebuild_relation does table_close() on OldHeap */
-
- pgstat_progress_end_command();
}
/*
@@ -1632,9 +1635,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
/* Cluster each relation in a separate transaction */
static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
{
ListCell *lc;
+ off_t childnum = 0;
/* Commit to get out of starting transaction */
PopActiveSnapshot();
@@ -1657,6 +1661,11 @@ cluster_multiple_rels(List *rvs, int options)
cluster_rel(rvtc->tableOid, rvtc->indexOid,
&cluster_params);
+ if (ispartitioned)
+ pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+ ++childnum);
+
+
PopActiveSnapshot();
CommitTransactionCommand();
}
diff --git a/src/backend/commands/vacuum.c b/src/backend/commands/vacuum.c
index c064352e23..05d52858b1 100644
--- a/src/backend/commands/vacuum.c
+++ b/src/backend/commands/vacuum.c
@@ -37,6 +37,7 @@
#include "catalog/pg_namespace.h"
#include "commands/cluster.h"
#include "commands/defrem.h"
+#include "commands/progress.h"
#include "commands/vacuum.h"
#include "miscadmin.h"
#include "nodes/makefuncs.h"
@@ -1937,7 +1938,11 @@ vacuum_rel(Oid relid, RangeVar *relation, VacuumParams *params)
cluster_params.options |= CLUOPT_VERBOSE;
/* VACUUM FULL is now a variant of CLUSTER; see cluster.c */
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, relid);
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
cluster_rel(relid, InvalidOid, &cluster_params);
+ pgstat_progress_end_command();
}
else
table_relation_vacuum(onerel, params, vac_strategy);
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
#define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS 5
#define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED 6
#define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT 7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL 17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE 18
/* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
#define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP 1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b1c9b7bdfe..c888415235 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1943,7 +1943,9 @@ pg_stat_progress_cluster| SELECT s.pid,
s.param5 AS heap_tuples_written,
s.param6 AS heap_blks_total,
s.param7 AS heap_blks_scanned,
- s.param8 AS index_rebuild_count
+ s.param8 AS index_rebuild_count,
+ s.param18 AS partitions_total,
+ s.param19 AS partitions_done
FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
LEFT JOIN pg_database d ON ((s.datid = d.oid)));
pg_stat_progress_copy| SELECT s.pid,
--
2.17.0
--fmvA4kSBHQVZhkR6
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v9-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"
^ permalink raw reply [nested|flat] 73+ messages in thread
* [PATCH v9 3/8] f! progress reporting..
@ 2021-02-09 02:45 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 73+ messages in thread
From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)
This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.
This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
doc/src/sgml/monitoring.sgml | 21 +++++++++++
src/backend/catalog/system_views.sql | 4 +-
src/backend/commands/cluster.c | 55 ++++++++++++++++------------
src/backend/commands/vacuum.c | 5 +++
src/include/commands/progress.h | 3 ++
src/test/regress/expected/rules.out | 4 +-
6 files changed, 67 insertions(+), 25 deletions(-)
diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml
index 3513e127b7..dca47f6a48 100644
--- a/doc/src/sgml/monitoring.sgml
+++ b/doc/src/sgml/monitoring.sgml
@@ -6231,6 +6231,27 @@ SELECT pg_stat_get_backend_pid(s.backendid) AS pid,
is <literal>rebuilding index</literal>.
</para></entry>
</row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>total_partitions</structfield> <type>bigint</type>
+ </para>
+ When clustering a partitioned table, this column is set to the total
+ number of partitions to be clustered.
+ <para>
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>done_partitions</structfield> <type>bigint</type>
+ </para>
+ <para>
+ When clustering a partitioned table, this column is set to the number
+ of partitions which have been clustered.
+ </para></entry>
+ </row>
+
</tbody>
</tgroup>
</table>
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fc94a73a54..21c27f0437 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1070,7 +1070,9 @@ CREATE VIEW pg_stat_progress_cluster AS
S.param5 AS heap_tuples_written,
S.param6 AS heap_blks_total,
S.param7 AS heap_blks_scanned,
- S.param8 AS index_rebuild_count
+ S.param8 AS index_rebuild_count,
+ S.param18 AS partitions_total,
+ S.param19 AS partitions_done
FROM pg_stat_get_progress_info('CLUSTER') AS S
LEFT JOIN pg_database D ON S.datid = D.oid;
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 0c08ac56dc..7d8d1d8a69 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
static List *get_tables_to_cluster(MemoryContext cluster_context);
static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
/*---------------------------------------------------------------------------
@@ -188,16 +188,26 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
/* close relation, keep lock till commit */
table_close(rel, ShareUpdateExclusiveLock);
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
{
/* Do the job. */
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_CLUSTER);
cluster_rel(tableOid, indexOid, ¶ms);
+ pgstat_progress_end_command();
}
else
{
List *rvs;
MemoryContext cluster_context;
+ int64 progress_val[2];
+ const int progress_index[2] = {
+ PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+ };
+
/* Refuse to hold strong locks in a user transaction */
PreventInTransactionBlock(isTopLevel, "CLUSTER");
@@ -206,7 +216,15 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
ALLOCSET_DEFAULT_SIZES);
rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
- cluster_multiple_rels(rvs, params.options);
+
+ /* Report command and number of partitions */
+ progress_val[0] = PROGRESS_CLUSTER_COMMAND_CLUSTER;
+ /* XXX: rvs includes the parent, which is not a "partition" */
+ progress_val[1] = list_length(rvs);
+
+ pgstat_progress_update_multi_param(2, progress_index, progress_val);
+ cluster_multiple_rels(rvs, params.options, true);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -245,7 +263,11 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
* cluster_context.
*/
rvs = get_tables_to_cluster(cluster_context);
- cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_CLUSTER);
+ cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -282,14 +304,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* Check for user-requested abort. */
CHECK_FOR_INTERRUPTS();
- pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
- if (OidIsValid(indexOid))
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_CLUSTER);
- else
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
/*
* We grab exclusive access to the target rel and index for the duration
* of the transaction. (This is redundant for the single-transaction
@@ -300,10 +314,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* If the table has gone away, we can skip processing it */
if (!OldHeap)
- {
- pgstat_progress_end_command();
return;
- }
/*
* Since we may open a new transaction for each relation, we have to check
@@ -319,7 +330,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!pg_class_ownercheck(tableOid, GetUserId()))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -334,7 +344,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (RELATION_IS_OTHER_TEMP(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -346,7 +355,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -357,7 +365,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!get_index_isclustered(indexOid))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
}
@@ -416,7 +423,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!RelationIsPopulated(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -424,7 +430,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -440,8 +445,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
rebuild_relation(OldHeap, indexOid, verbose);
/* NB: rebuild_relation does table_close() on OldHeap */
-
- pgstat_progress_end_command();
}
/*
@@ -1632,9 +1635,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
/* Cluster each relation in a separate transaction */
static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
{
ListCell *lc;
+ off_t childnum = 0;
/* Commit to get out of starting transaction */
PopActiveSnapshot();
@@ -1657,6 +1661,11 @@ cluster_multiple_rels(List *rvs, int options)
cluster_rel(rvtc->tableOid, rvtc->indexOid,
&cluster_params);
+ if (ispartitioned)
+ pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+ ++childnum);
+
+
PopActiveSnapshot();
CommitTransactionCommand();
}
diff --git a/src/backend/commands/vacuum.c b/src/backend/commands/vacuum.c
index c064352e23..05d52858b1 100644
--- a/src/backend/commands/vacuum.c
+++ b/src/backend/commands/vacuum.c
@@ -37,6 +37,7 @@
#include "catalog/pg_namespace.h"
#include "commands/cluster.h"
#include "commands/defrem.h"
+#include "commands/progress.h"
#include "commands/vacuum.h"
#include "miscadmin.h"
#include "nodes/makefuncs.h"
@@ -1937,7 +1938,11 @@ vacuum_rel(Oid relid, RangeVar *relation, VacuumParams *params)
cluster_params.options |= CLUOPT_VERBOSE;
/* VACUUM FULL is now a variant of CLUSTER; see cluster.c */
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, relid);
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
cluster_rel(relid, InvalidOid, &cluster_params);
+ pgstat_progress_end_command();
}
else
table_relation_vacuum(onerel, params, vac_strategy);
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
#define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS 5
#define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED 6
#define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT 7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL 17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE 18
/* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
#define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP 1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b1c9b7bdfe..c888415235 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1943,7 +1943,9 @@ pg_stat_progress_cluster| SELECT s.pid,
s.param5 AS heap_tuples_written,
s.param6 AS heap_blks_total,
s.param7 AS heap_blks_scanned,
- s.param8 AS index_rebuild_count
+ s.param8 AS index_rebuild_count,
+ s.param18 AS partitions_total,
+ s.param19 AS partitions_done
FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
LEFT JOIN pg_database d ON ((s.datid = d.oid)));
pg_stat_progress_copy| SELECT s.pid,
--
2.17.0
--fmvA4kSBHQVZhkR6
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v9-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"
^ permalink raw reply [nested|flat] 73+ messages in thread
* [PATCH v9 3/8] f! progress reporting..
@ 2021-02-09 02:45 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 73+ messages in thread
From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)
This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.
This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
doc/src/sgml/monitoring.sgml | 21 +++++++++++
src/backend/catalog/system_views.sql | 4 +-
src/backend/commands/cluster.c | 55 ++++++++++++++++------------
src/backend/commands/vacuum.c | 5 +++
src/include/commands/progress.h | 3 ++
src/test/regress/expected/rules.out | 4 +-
6 files changed, 67 insertions(+), 25 deletions(-)
diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml
index 3513e127b7..dca47f6a48 100644
--- a/doc/src/sgml/monitoring.sgml
+++ b/doc/src/sgml/monitoring.sgml
@@ -6231,6 +6231,27 @@ SELECT pg_stat_get_backend_pid(s.backendid) AS pid,
is <literal>rebuilding index</literal>.
</para></entry>
</row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>total_partitions</structfield> <type>bigint</type>
+ </para>
+ When clustering a partitioned table, this column is set to the total
+ number of partitions to be clustered.
+ <para>
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>done_partitions</structfield> <type>bigint</type>
+ </para>
+ <para>
+ When clustering a partitioned table, this column is set to the number
+ of partitions which have been clustered.
+ </para></entry>
+ </row>
+
</tbody>
</tgroup>
</table>
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fc94a73a54..21c27f0437 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1070,7 +1070,9 @@ CREATE VIEW pg_stat_progress_cluster AS
S.param5 AS heap_tuples_written,
S.param6 AS heap_blks_total,
S.param7 AS heap_blks_scanned,
- S.param8 AS index_rebuild_count
+ S.param8 AS index_rebuild_count,
+ S.param18 AS partitions_total,
+ S.param19 AS partitions_done
FROM pg_stat_get_progress_info('CLUSTER') AS S
LEFT JOIN pg_database D ON S.datid = D.oid;
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 0c08ac56dc..7d8d1d8a69 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
static List *get_tables_to_cluster(MemoryContext cluster_context);
static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
/*---------------------------------------------------------------------------
@@ -188,16 +188,26 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
/* close relation, keep lock till commit */
table_close(rel, ShareUpdateExclusiveLock);
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
{
/* Do the job. */
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_CLUSTER);
cluster_rel(tableOid, indexOid, ¶ms);
+ pgstat_progress_end_command();
}
else
{
List *rvs;
MemoryContext cluster_context;
+ int64 progress_val[2];
+ const int progress_index[2] = {
+ PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+ };
+
/* Refuse to hold strong locks in a user transaction */
PreventInTransactionBlock(isTopLevel, "CLUSTER");
@@ -206,7 +216,15 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
ALLOCSET_DEFAULT_SIZES);
rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
- cluster_multiple_rels(rvs, params.options);
+
+ /* Report command and number of partitions */
+ progress_val[0] = PROGRESS_CLUSTER_COMMAND_CLUSTER;
+ /* XXX: rvs includes the parent, which is not a "partition" */
+ progress_val[1] = list_length(rvs);
+
+ pgstat_progress_update_multi_param(2, progress_index, progress_val);
+ cluster_multiple_rels(rvs, params.options, true);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -245,7 +263,11 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
* cluster_context.
*/
rvs = get_tables_to_cluster(cluster_context);
- cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_CLUSTER);
+ cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -282,14 +304,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* Check for user-requested abort. */
CHECK_FOR_INTERRUPTS();
- pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
- if (OidIsValid(indexOid))
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_CLUSTER);
- else
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
/*
* We grab exclusive access to the target rel and index for the duration
* of the transaction. (This is redundant for the single-transaction
@@ -300,10 +314,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* If the table has gone away, we can skip processing it */
if (!OldHeap)
- {
- pgstat_progress_end_command();
return;
- }
/*
* Since we may open a new transaction for each relation, we have to check
@@ -319,7 +330,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!pg_class_ownercheck(tableOid, GetUserId()))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -334,7 +344,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (RELATION_IS_OTHER_TEMP(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -346,7 +355,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -357,7 +365,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!get_index_isclustered(indexOid))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
}
@@ -416,7 +423,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!RelationIsPopulated(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -424,7 +430,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -440,8 +445,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
rebuild_relation(OldHeap, indexOid, verbose);
/* NB: rebuild_relation does table_close() on OldHeap */
-
- pgstat_progress_end_command();
}
/*
@@ -1632,9 +1635,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
/* Cluster each relation in a separate transaction */
static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
{
ListCell *lc;
+ off_t childnum = 0;
/* Commit to get out of starting transaction */
PopActiveSnapshot();
@@ -1657,6 +1661,11 @@ cluster_multiple_rels(List *rvs, int options)
cluster_rel(rvtc->tableOid, rvtc->indexOid,
&cluster_params);
+ if (ispartitioned)
+ pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+ ++childnum);
+
+
PopActiveSnapshot();
CommitTransactionCommand();
}
diff --git a/src/backend/commands/vacuum.c b/src/backend/commands/vacuum.c
index c064352e23..05d52858b1 100644
--- a/src/backend/commands/vacuum.c
+++ b/src/backend/commands/vacuum.c
@@ -37,6 +37,7 @@
#include "catalog/pg_namespace.h"
#include "commands/cluster.h"
#include "commands/defrem.h"
+#include "commands/progress.h"
#include "commands/vacuum.h"
#include "miscadmin.h"
#include "nodes/makefuncs.h"
@@ -1937,7 +1938,11 @@ vacuum_rel(Oid relid, RangeVar *relation, VacuumParams *params)
cluster_params.options |= CLUOPT_VERBOSE;
/* VACUUM FULL is now a variant of CLUSTER; see cluster.c */
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, relid);
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
cluster_rel(relid, InvalidOid, &cluster_params);
+ pgstat_progress_end_command();
}
else
table_relation_vacuum(onerel, params, vac_strategy);
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
#define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS 5
#define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED 6
#define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT 7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL 17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE 18
/* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
#define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP 1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b1c9b7bdfe..c888415235 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1943,7 +1943,9 @@ pg_stat_progress_cluster| SELECT s.pid,
s.param5 AS heap_tuples_written,
s.param6 AS heap_blks_total,
s.param7 AS heap_blks_scanned,
- s.param8 AS index_rebuild_count
+ s.param8 AS index_rebuild_count,
+ s.param18 AS partitions_total,
+ s.param19 AS partitions_done
FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
LEFT JOIN pg_database d ON ((s.datid = d.oid)));
pg_stat_progress_copy| SELECT s.pid,
--
2.17.0
--fmvA4kSBHQVZhkR6
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v9-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"
^ permalink raw reply [nested|flat] 73+ messages in thread
* [PATCH v8 3/8] f! progress reporting..
@ 2021-02-09 02:45 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 73+ messages in thread
From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)
This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.
This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
src/backend/catalog/system_views.sql | 4 +-
src/backend/commands/cluster.c | 64 +++++++++++++++++-----------
src/include/commands/progress.h | 3 ++
src/test/regress/expected/rules.out | 4 +-
4 files changed, 47 insertions(+), 28 deletions(-)
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fa58afd9d7..9a3cb5f935 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1069,7 +1069,9 @@ CREATE VIEW pg_stat_progress_cluster AS
S.param5 AS heap_tuples_written,
S.param6 AS heap_blks_total,
S.param7 AS heap_blks_scanned,
- S.param8 AS index_rebuild_count
+ S.param8 AS index_rebuild_count,
+ S.param18 AS partitions_total,
+ S.param19 AS partitions_done
FROM pg_stat_get_progress_info('CLUSTER') AS S
LEFT JOIN pg_database D ON S.datid = D.oid;
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 9b6673867c..cb4fc350c6 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
static List *get_tables_to_cluster(MemoryContext cluster_context);
static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
/*---------------------------------------------------------------------------
@@ -188,16 +188,28 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
/* close relation, keep lock till commit */
table_close(rel, ShareUpdateExclusiveLock);
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
{
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ OidIsValid(indexOid) ? PROGRESS_CLUSTER_COMMAND_CLUSTER :
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
+
/* Do the job. */
cluster_rel(tableOid, indexOid, ¶ms);
+ pgstat_progress_end_command();
}
else
{
List *rvs;
MemoryContext cluster_context;
+ int64 progress_val[2];
+ const int progress_index[] = {
+ PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+ };
+
/* Refuse to hold strong locks in a user transaction */
PreventInTransactionBlock(isTopLevel, "CLUSTER");
@@ -206,7 +218,20 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
ALLOCSET_DEFAULT_SIZES);
rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
- cluster_multiple_rels(rvs, params.options);
+
+ /* Report vacuum full or cluster and number of partitions */
+ progress_val[0] = OidIsValid(indexOid) ?
+ PROGRESS_CLUSTER_COMMAND_CLUSTER :
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL;
+ progress_val[1] = list_length(rvs);
+// This is currently showing the total number of *tables*, including the
+// parent, and intermediate partitions, so the column should be renamed, or
+// we should count the number of leaf partitions
+
+ pgstat_progress_update_multi_param(2, progress_index, progress_val);
+
+ cluster_multiple_rels(rvs, params.options, true);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -245,7 +270,7 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
* cluster_context.
*/
rvs = get_tables_to_cluster(cluster_context);
- cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+ cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -282,14 +307,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* Check for user-requested abort. */
CHECK_FOR_INTERRUPTS();
- pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
- if (OidIsValid(indexOid))
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_CLUSTER);
- else
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
/*
* We grab exclusive access to the target rel and index for the duration
* of the transaction. (This is redundant for the single-transaction
@@ -300,10 +317,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* If the table has gone away, we can skip processing it */
if (!OldHeap)
- {
- pgstat_progress_end_command();
return;
- }
/*
* Since we may open a new transaction for each relation, we have to check
@@ -319,7 +333,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!pg_class_ownercheck(tableOid, GetUserId()))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -334,7 +347,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (RELATION_IS_OTHER_TEMP(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -346,7 +358,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -357,7 +368,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!get_index_isclustered(indexOid))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
}
@@ -416,7 +426,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!RelationIsPopulated(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -424,7 +433,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -440,8 +448,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
rebuild_relation(OldHeap, indexOid, verbose);
/* NB: rebuild_relation does table_close() on OldHeap */
-
- pgstat_progress_end_command();
}
/*
@@ -1366,7 +1372,7 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap,
ReindexParams reindex_params = {0};
int i;
- /* Report that we are now swapping relation files */
+ /* Report that we are now swapping relation files XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_SWAP_REL_FILES);
@@ -1417,13 +1423,13 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap,
else if (newrelpersistence == RELPERSISTENCE_PERMANENT)
reindex_flags |= REINDEX_REL_FORCE_INDEXES_PERMANENT;
- /* Report that we are now reindexing relations */
+ /* Report that we are now reindexing relations XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_REBUILD_INDEX);
reindex_relation(OIDOldHeap, reindex_flags, &reindex_params);
- /* Report that we are now doing clean up */
+ /* Report that we are now doing clean up XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_FINAL_CLEANUP);
@@ -1633,9 +1639,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
/* Cluster each relation in a separate transaction */
static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
{
ListCell *lc;
+ off_t childnum = 0;
/* Commit to get out of starting transaction */
PopActiveSnapshot();
@@ -1658,6 +1665,11 @@ cluster_multiple_rels(List *rvs, int options)
cluster_rel(rvtc->tableOid, rvtc->indexOid,
&cluster_params);
+ if (ispartitioned)
+ pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+ childnum++);
+
+
PopActiveSnapshot();
CommitTransactionCommand();
}
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
#define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS 5
#define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED 6
#define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT 7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL 17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE 18
/* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
#define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP 1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b632d9f2ea..d1fea532bc 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1941,7 +1941,9 @@ pg_stat_progress_cluster| SELECT s.pid,
s.param5 AS heap_tuples_written,
s.param6 AS heap_blks_total,
s.param7 AS heap_blks_scanned,
- s.param8 AS index_rebuild_count
+ s.param8 AS index_rebuild_count,
+ s.param18 AS partitions_total,
+ s.param19 AS partitions_done
FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
LEFT JOIN pg_database d ON ((s.datid = d.oid)));
pg_stat_progress_copy| SELECT s.pid,
--
2.17.0
--O5XBE6gyVG5Rl6Rj
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v8-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"
^ permalink raw reply [nested|flat] 73+ messages in thread
* [PATCH v8 3/8] f! progress reporting..
@ 2021-02-09 02:45 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 73+ messages in thread
From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)
This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.
This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
src/backend/catalog/system_views.sql | 4 +-
src/backend/commands/cluster.c | 64 +++++++++++++++++-----------
src/include/commands/progress.h | 3 ++
src/test/regress/expected/rules.out | 4 +-
4 files changed, 47 insertions(+), 28 deletions(-)
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fa58afd9d7..9a3cb5f935 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1069,7 +1069,9 @@ CREATE VIEW pg_stat_progress_cluster AS
S.param5 AS heap_tuples_written,
S.param6 AS heap_blks_total,
S.param7 AS heap_blks_scanned,
- S.param8 AS index_rebuild_count
+ S.param8 AS index_rebuild_count,
+ S.param18 AS partitions_total,
+ S.param19 AS partitions_done
FROM pg_stat_get_progress_info('CLUSTER') AS S
LEFT JOIN pg_database D ON S.datid = D.oid;
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 9b6673867c..cb4fc350c6 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
static List *get_tables_to_cluster(MemoryContext cluster_context);
static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
/*---------------------------------------------------------------------------
@@ -188,16 +188,28 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
/* close relation, keep lock till commit */
table_close(rel, ShareUpdateExclusiveLock);
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
{
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ OidIsValid(indexOid) ? PROGRESS_CLUSTER_COMMAND_CLUSTER :
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
+
/* Do the job. */
cluster_rel(tableOid, indexOid, ¶ms);
+ pgstat_progress_end_command();
}
else
{
List *rvs;
MemoryContext cluster_context;
+ int64 progress_val[2];
+ const int progress_index[] = {
+ PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+ };
+
/* Refuse to hold strong locks in a user transaction */
PreventInTransactionBlock(isTopLevel, "CLUSTER");
@@ -206,7 +218,20 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
ALLOCSET_DEFAULT_SIZES);
rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
- cluster_multiple_rels(rvs, params.options);
+
+ /* Report vacuum full or cluster and number of partitions */
+ progress_val[0] = OidIsValid(indexOid) ?
+ PROGRESS_CLUSTER_COMMAND_CLUSTER :
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL;
+ progress_val[1] = list_length(rvs);
+// This is currently showing the total number of *tables*, including the
+// parent, and intermediate partitions, so the column should be renamed, or
+// we should count the number of leaf partitions
+
+ pgstat_progress_update_multi_param(2, progress_index, progress_val);
+
+ cluster_multiple_rels(rvs, params.options, true);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -245,7 +270,7 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
* cluster_context.
*/
rvs = get_tables_to_cluster(cluster_context);
- cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+ cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -282,14 +307,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* Check for user-requested abort. */
CHECK_FOR_INTERRUPTS();
- pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
- if (OidIsValid(indexOid))
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_CLUSTER);
- else
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
/*
* We grab exclusive access to the target rel and index for the duration
* of the transaction. (This is redundant for the single-transaction
@@ -300,10 +317,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* If the table has gone away, we can skip processing it */
if (!OldHeap)
- {
- pgstat_progress_end_command();
return;
- }
/*
* Since we may open a new transaction for each relation, we have to check
@@ -319,7 +333,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!pg_class_ownercheck(tableOid, GetUserId()))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -334,7 +347,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (RELATION_IS_OTHER_TEMP(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -346,7 +358,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -357,7 +368,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!get_index_isclustered(indexOid))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
}
@@ -416,7 +426,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!RelationIsPopulated(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -424,7 +433,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -440,8 +448,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
rebuild_relation(OldHeap, indexOid, verbose);
/* NB: rebuild_relation does table_close() on OldHeap */
-
- pgstat_progress_end_command();
}
/*
@@ -1366,7 +1372,7 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap,
ReindexParams reindex_params = {0};
int i;
- /* Report that we are now swapping relation files */
+ /* Report that we are now swapping relation files XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_SWAP_REL_FILES);
@@ -1417,13 +1423,13 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap,
else if (newrelpersistence == RELPERSISTENCE_PERMANENT)
reindex_flags |= REINDEX_REL_FORCE_INDEXES_PERMANENT;
- /* Report that we are now reindexing relations */
+ /* Report that we are now reindexing relations XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_REBUILD_INDEX);
reindex_relation(OIDOldHeap, reindex_flags, &reindex_params);
- /* Report that we are now doing clean up */
+ /* Report that we are now doing clean up XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_FINAL_CLEANUP);
@@ -1633,9 +1639,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
/* Cluster each relation in a separate transaction */
static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
{
ListCell *lc;
+ off_t childnum = 0;
/* Commit to get out of starting transaction */
PopActiveSnapshot();
@@ -1658,6 +1665,11 @@ cluster_multiple_rels(List *rvs, int options)
cluster_rel(rvtc->tableOid, rvtc->indexOid,
&cluster_params);
+ if (ispartitioned)
+ pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+ childnum++);
+
+
PopActiveSnapshot();
CommitTransactionCommand();
}
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
#define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS 5
#define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED 6
#define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT 7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL 17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE 18
/* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
#define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP 1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b632d9f2ea..d1fea532bc 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1941,7 +1941,9 @@ pg_stat_progress_cluster| SELECT s.pid,
s.param5 AS heap_tuples_written,
s.param6 AS heap_blks_total,
s.param7 AS heap_blks_scanned,
- s.param8 AS index_rebuild_count
+ s.param8 AS index_rebuild_count,
+ s.param18 AS partitions_total,
+ s.param19 AS partitions_done
FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
LEFT JOIN pg_database d ON ((s.datid = d.oid)));
pg_stat_progress_copy| SELECT s.pid,
--
2.17.0
--O5XBE6gyVG5Rl6Rj
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v8-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"
^ permalink raw reply [nested|flat] 73+ messages in thread
* [PATCH v8 3/8] f! progress reporting..
@ 2021-02-09 02:45 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 73+ messages in thread
From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)
This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.
This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
src/backend/catalog/system_views.sql | 4 +-
src/backend/commands/cluster.c | 64 +++++++++++++++++-----------
src/include/commands/progress.h | 3 ++
src/test/regress/expected/rules.out | 4 +-
4 files changed, 47 insertions(+), 28 deletions(-)
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fa58afd9d7..9a3cb5f935 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1069,7 +1069,9 @@ CREATE VIEW pg_stat_progress_cluster AS
S.param5 AS heap_tuples_written,
S.param6 AS heap_blks_total,
S.param7 AS heap_blks_scanned,
- S.param8 AS index_rebuild_count
+ S.param8 AS index_rebuild_count,
+ S.param18 AS partitions_total,
+ S.param19 AS partitions_done
FROM pg_stat_get_progress_info('CLUSTER') AS S
LEFT JOIN pg_database D ON S.datid = D.oid;
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 9b6673867c..cb4fc350c6 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
static List *get_tables_to_cluster(MemoryContext cluster_context);
static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
/*---------------------------------------------------------------------------
@@ -188,16 +188,28 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
/* close relation, keep lock till commit */
table_close(rel, ShareUpdateExclusiveLock);
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
{
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ OidIsValid(indexOid) ? PROGRESS_CLUSTER_COMMAND_CLUSTER :
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
+
/* Do the job. */
cluster_rel(tableOid, indexOid, ¶ms);
+ pgstat_progress_end_command();
}
else
{
List *rvs;
MemoryContext cluster_context;
+ int64 progress_val[2];
+ const int progress_index[] = {
+ PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+ };
+
/* Refuse to hold strong locks in a user transaction */
PreventInTransactionBlock(isTopLevel, "CLUSTER");
@@ -206,7 +218,20 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
ALLOCSET_DEFAULT_SIZES);
rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
- cluster_multiple_rels(rvs, params.options);
+
+ /* Report vacuum full or cluster and number of partitions */
+ progress_val[0] = OidIsValid(indexOid) ?
+ PROGRESS_CLUSTER_COMMAND_CLUSTER :
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL;
+ progress_val[1] = list_length(rvs);
+// This is currently showing the total number of *tables*, including the
+// parent, and intermediate partitions, so the column should be renamed, or
+// we should count the number of leaf partitions
+
+ pgstat_progress_update_multi_param(2, progress_index, progress_val);
+
+ cluster_multiple_rels(rvs, params.options, true);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -245,7 +270,7 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
* cluster_context.
*/
rvs = get_tables_to_cluster(cluster_context);
- cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+ cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -282,14 +307,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* Check for user-requested abort. */
CHECK_FOR_INTERRUPTS();
- pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
- if (OidIsValid(indexOid))
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_CLUSTER);
- else
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
/*
* We grab exclusive access to the target rel and index for the duration
* of the transaction. (This is redundant for the single-transaction
@@ -300,10 +317,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* If the table has gone away, we can skip processing it */
if (!OldHeap)
- {
- pgstat_progress_end_command();
return;
- }
/*
* Since we may open a new transaction for each relation, we have to check
@@ -319,7 +333,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!pg_class_ownercheck(tableOid, GetUserId()))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -334,7 +347,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (RELATION_IS_OTHER_TEMP(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -346,7 +358,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -357,7 +368,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!get_index_isclustered(indexOid))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
}
@@ -416,7 +426,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!RelationIsPopulated(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -424,7 +433,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -440,8 +448,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
rebuild_relation(OldHeap, indexOid, verbose);
/* NB: rebuild_relation does table_close() on OldHeap */
-
- pgstat_progress_end_command();
}
/*
@@ -1366,7 +1372,7 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap,
ReindexParams reindex_params = {0};
int i;
- /* Report that we are now swapping relation files */
+ /* Report that we are now swapping relation files XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_SWAP_REL_FILES);
@@ -1417,13 +1423,13 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap,
else if (newrelpersistence == RELPERSISTENCE_PERMANENT)
reindex_flags |= REINDEX_REL_FORCE_INDEXES_PERMANENT;
- /* Report that we are now reindexing relations */
+ /* Report that we are now reindexing relations XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_REBUILD_INDEX);
reindex_relation(OIDOldHeap, reindex_flags, &reindex_params);
- /* Report that we are now doing clean up */
+ /* Report that we are now doing clean up XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_FINAL_CLEANUP);
@@ -1633,9 +1639,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
/* Cluster each relation in a separate transaction */
static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
{
ListCell *lc;
+ off_t childnum = 0;
/* Commit to get out of starting transaction */
PopActiveSnapshot();
@@ -1658,6 +1665,11 @@ cluster_multiple_rels(List *rvs, int options)
cluster_rel(rvtc->tableOid, rvtc->indexOid,
&cluster_params);
+ if (ispartitioned)
+ pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+ childnum++);
+
+
PopActiveSnapshot();
CommitTransactionCommand();
}
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
#define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS 5
#define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED 6
#define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT 7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL 17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE 18
/* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
#define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP 1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b632d9f2ea..d1fea532bc 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1941,7 +1941,9 @@ pg_stat_progress_cluster| SELECT s.pid,
s.param5 AS heap_tuples_written,
s.param6 AS heap_blks_total,
s.param7 AS heap_blks_scanned,
- s.param8 AS index_rebuild_count
+ s.param8 AS index_rebuild_count,
+ s.param18 AS partitions_total,
+ s.param19 AS partitions_done
FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
LEFT JOIN pg_database d ON ((s.datid = d.oid)));
pg_stat_progress_copy| SELECT s.pid,
--
2.17.0
--O5XBE6gyVG5Rl6Rj
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v8-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"
^ permalink raw reply [nested|flat] 73+ messages in thread
* [PATCH v8 3/8] f! progress reporting..
@ 2021-02-09 02:45 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 73+ messages in thread
From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)
This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.
This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
src/backend/catalog/system_views.sql | 4 +-
src/backend/commands/cluster.c | 64 +++++++++++++++++-----------
src/include/commands/progress.h | 3 ++
src/test/regress/expected/rules.out | 4 +-
4 files changed, 47 insertions(+), 28 deletions(-)
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fa58afd9d7..9a3cb5f935 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1069,7 +1069,9 @@ CREATE VIEW pg_stat_progress_cluster AS
S.param5 AS heap_tuples_written,
S.param6 AS heap_blks_total,
S.param7 AS heap_blks_scanned,
- S.param8 AS index_rebuild_count
+ S.param8 AS index_rebuild_count,
+ S.param18 AS partitions_total,
+ S.param19 AS partitions_done
FROM pg_stat_get_progress_info('CLUSTER') AS S
LEFT JOIN pg_database D ON S.datid = D.oid;
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 9b6673867c..cb4fc350c6 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
static List *get_tables_to_cluster(MemoryContext cluster_context);
static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
/*---------------------------------------------------------------------------
@@ -188,16 +188,28 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
/* close relation, keep lock till commit */
table_close(rel, ShareUpdateExclusiveLock);
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
{
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ OidIsValid(indexOid) ? PROGRESS_CLUSTER_COMMAND_CLUSTER :
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
+
/* Do the job. */
cluster_rel(tableOid, indexOid, ¶ms);
+ pgstat_progress_end_command();
}
else
{
List *rvs;
MemoryContext cluster_context;
+ int64 progress_val[2];
+ const int progress_index[] = {
+ PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+ };
+
/* Refuse to hold strong locks in a user transaction */
PreventInTransactionBlock(isTopLevel, "CLUSTER");
@@ -206,7 +218,20 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
ALLOCSET_DEFAULT_SIZES);
rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
- cluster_multiple_rels(rvs, params.options);
+
+ /* Report vacuum full or cluster and number of partitions */
+ progress_val[0] = OidIsValid(indexOid) ?
+ PROGRESS_CLUSTER_COMMAND_CLUSTER :
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL;
+ progress_val[1] = list_length(rvs);
+// This is currently showing the total number of *tables*, including the
+// parent, and intermediate partitions, so the column should be renamed, or
+// we should count the number of leaf partitions
+
+ pgstat_progress_update_multi_param(2, progress_index, progress_val);
+
+ cluster_multiple_rels(rvs, params.options, true);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -245,7 +270,7 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
* cluster_context.
*/
rvs = get_tables_to_cluster(cluster_context);
- cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+ cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -282,14 +307,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* Check for user-requested abort. */
CHECK_FOR_INTERRUPTS();
- pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
- if (OidIsValid(indexOid))
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_CLUSTER);
- else
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
/*
* We grab exclusive access to the target rel and index for the duration
* of the transaction. (This is redundant for the single-transaction
@@ -300,10 +317,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* If the table has gone away, we can skip processing it */
if (!OldHeap)
- {
- pgstat_progress_end_command();
return;
- }
/*
* Since we may open a new transaction for each relation, we have to check
@@ -319,7 +333,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!pg_class_ownercheck(tableOid, GetUserId()))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -334,7 +347,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (RELATION_IS_OTHER_TEMP(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -346,7 +358,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -357,7 +368,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!get_index_isclustered(indexOid))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
}
@@ -416,7 +426,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!RelationIsPopulated(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -424,7 +433,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -440,8 +448,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
rebuild_relation(OldHeap, indexOid, verbose);
/* NB: rebuild_relation does table_close() on OldHeap */
-
- pgstat_progress_end_command();
}
/*
@@ -1366,7 +1372,7 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap,
ReindexParams reindex_params = {0};
int i;
- /* Report that we are now swapping relation files */
+ /* Report that we are now swapping relation files XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_SWAP_REL_FILES);
@@ -1417,13 +1423,13 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap,
else if (newrelpersistence == RELPERSISTENCE_PERMANENT)
reindex_flags |= REINDEX_REL_FORCE_INDEXES_PERMANENT;
- /* Report that we are now reindexing relations */
+ /* Report that we are now reindexing relations XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_REBUILD_INDEX);
reindex_relation(OIDOldHeap, reindex_flags, &reindex_params);
- /* Report that we are now doing clean up */
+ /* Report that we are now doing clean up XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_FINAL_CLEANUP);
@@ -1633,9 +1639,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
/* Cluster each relation in a separate transaction */
static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
{
ListCell *lc;
+ off_t childnum = 0;
/* Commit to get out of starting transaction */
PopActiveSnapshot();
@@ -1658,6 +1665,11 @@ cluster_multiple_rels(List *rvs, int options)
cluster_rel(rvtc->tableOid, rvtc->indexOid,
&cluster_params);
+ if (ispartitioned)
+ pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+ childnum++);
+
+
PopActiveSnapshot();
CommitTransactionCommand();
}
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
#define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS 5
#define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED 6
#define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT 7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL 17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE 18
/* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
#define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP 1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b632d9f2ea..d1fea532bc 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1941,7 +1941,9 @@ pg_stat_progress_cluster| SELECT s.pid,
s.param5 AS heap_tuples_written,
s.param6 AS heap_blks_total,
s.param7 AS heap_blks_scanned,
- s.param8 AS index_rebuild_count
+ s.param8 AS index_rebuild_count,
+ s.param18 AS partitions_total,
+ s.param19 AS partitions_done
FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
LEFT JOIN pg_database d ON ((s.datid = d.oid)));
pg_stat_progress_copy| SELECT s.pid,
--
2.17.0
--O5XBE6gyVG5Rl6Rj
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v8-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"
^ permalink raw reply [nested|flat] 73+ messages in thread
* [PATCH v8 3/8] f! progress reporting..
@ 2021-02-09 02:45 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 73+ messages in thread
From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)
This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.
This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
src/backend/catalog/system_views.sql | 4 +-
src/backend/commands/cluster.c | 64 +++++++++++++++++-----------
src/include/commands/progress.h | 3 ++
src/test/regress/expected/rules.out | 4 +-
4 files changed, 47 insertions(+), 28 deletions(-)
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fa58afd9d7..9a3cb5f935 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1069,7 +1069,9 @@ CREATE VIEW pg_stat_progress_cluster AS
S.param5 AS heap_tuples_written,
S.param6 AS heap_blks_total,
S.param7 AS heap_blks_scanned,
- S.param8 AS index_rebuild_count
+ S.param8 AS index_rebuild_count,
+ S.param18 AS partitions_total,
+ S.param19 AS partitions_done
FROM pg_stat_get_progress_info('CLUSTER') AS S
LEFT JOIN pg_database D ON S.datid = D.oid;
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 9b6673867c..cb4fc350c6 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
static List *get_tables_to_cluster(MemoryContext cluster_context);
static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
/*---------------------------------------------------------------------------
@@ -188,16 +188,28 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
/* close relation, keep lock till commit */
table_close(rel, ShareUpdateExclusiveLock);
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
{
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ OidIsValid(indexOid) ? PROGRESS_CLUSTER_COMMAND_CLUSTER :
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
+
/* Do the job. */
cluster_rel(tableOid, indexOid, ¶ms);
+ pgstat_progress_end_command();
}
else
{
List *rvs;
MemoryContext cluster_context;
+ int64 progress_val[2];
+ const int progress_index[] = {
+ PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+ };
+
/* Refuse to hold strong locks in a user transaction */
PreventInTransactionBlock(isTopLevel, "CLUSTER");
@@ -206,7 +218,20 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
ALLOCSET_DEFAULT_SIZES);
rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
- cluster_multiple_rels(rvs, params.options);
+
+ /* Report vacuum full or cluster and number of partitions */
+ progress_val[0] = OidIsValid(indexOid) ?
+ PROGRESS_CLUSTER_COMMAND_CLUSTER :
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL;
+ progress_val[1] = list_length(rvs);
+// This is currently showing the total number of *tables*, including the
+// parent, and intermediate partitions, so the column should be renamed, or
+// we should count the number of leaf partitions
+
+ pgstat_progress_update_multi_param(2, progress_index, progress_val);
+
+ cluster_multiple_rels(rvs, params.options, true);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -245,7 +270,7 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
* cluster_context.
*/
rvs = get_tables_to_cluster(cluster_context);
- cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+ cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -282,14 +307,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* Check for user-requested abort. */
CHECK_FOR_INTERRUPTS();
- pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
- if (OidIsValid(indexOid))
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_CLUSTER);
- else
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
/*
* We grab exclusive access to the target rel and index for the duration
* of the transaction. (This is redundant for the single-transaction
@@ -300,10 +317,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* If the table has gone away, we can skip processing it */
if (!OldHeap)
- {
- pgstat_progress_end_command();
return;
- }
/*
* Since we may open a new transaction for each relation, we have to check
@@ -319,7 +333,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!pg_class_ownercheck(tableOid, GetUserId()))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -334,7 +347,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (RELATION_IS_OTHER_TEMP(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -346,7 +358,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -357,7 +368,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!get_index_isclustered(indexOid))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
}
@@ -416,7 +426,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!RelationIsPopulated(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -424,7 +433,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -440,8 +448,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
rebuild_relation(OldHeap, indexOid, verbose);
/* NB: rebuild_relation does table_close() on OldHeap */
-
- pgstat_progress_end_command();
}
/*
@@ -1366,7 +1372,7 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap,
ReindexParams reindex_params = {0};
int i;
- /* Report that we are now swapping relation files */
+ /* Report that we are now swapping relation files XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_SWAP_REL_FILES);
@@ -1417,13 +1423,13 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap,
else if (newrelpersistence == RELPERSISTENCE_PERMANENT)
reindex_flags |= REINDEX_REL_FORCE_INDEXES_PERMANENT;
- /* Report that we are now reindexing relations */
+ /* Report that we are now reindexing relations XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_REBUILD_INDEX);
reindex_relation(OIDOldHeap, reindex_flags, &reindex_params);
- /* Report that we are now doing clean up */
+ /* Report that we are now doing clean up XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_FINAL_CLEANUP);
@@ -1633,9 +1639,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
/* Cluster each relation in a separate transaction */
static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
{
ListCell *lc;
+ off_t childnum = 0;
/* Commit to get out of starting transaction */
PopActiveSnapshot();
@@ -1658,6 +1665,11 @@ cluster_multiple_rels(List *rvs, int options)
cluster_rel(rvtc->tableOid, rvtc->indexOid,
&cluster_params);
+ if (ispartitioned)
+ pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+ childnum++);
+
+
PopActiveSnapshot();
CommitTransactionCommand();
}
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
#define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS 5
#define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED 6
#define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT 7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL 17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE 18
/* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
#define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP 1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b632d9f2ea..d1fea532bc 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1941,7 +1941,9 @@ pg_stat_progress_cluster| SELECT s.pid,
s.param5 AS heap_tuples_written,
s.param6 AS heap_blks_total,
s.param7 AS heap_blks_scanned,
- s.param8 AS index_rebuild_count
+ s.param8 AS index_rebuild_count,
+ s.param18 AS partitions_total,
+ s.param19 AS partitions_done
FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
LEFT JOIN pg_database d ON ((s.datid = d.oid)));
pg_stat_progress_copy| SELECT s.pid,
--
2.17.0
--O5XBE6gyVG5Rl6Rj
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v8-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"
^ permalink raw reply [nested|flat] 73+ messages in thread
* [PATCH v8 3/8] f! progress reporting..
@ 2021-02-09 02:45 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 73+ messages in thread
From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)
This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.
This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
src/backend/catalog/system_views.sql | 4 +-
src/backend/commands/cluster.c | 64 +++++++++++++++++-----------
src/include/commands/progress.h | 3 ++
src/test/regress/expected/rules.out | 4 +-
4 files changed, 47 insertions(+), 28 deletions(-)
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fa58afd9d7..9a3cb5f935 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1069,7 +1069,9 @@ CREATE VIEW pg_stat_progress_cluster AS
S.param5 AS heap_tuples_written,
S.param6 AS heap_blks_total,
S.param7 AS heap_blks_scanned,
- S.param8 AS index_rebuild_count
+ S.param8 AS index_rebuild_count,
+ S.param18 AS partitions_total,
+ S.param19 AS partitions_done
FROM pg_stat_get_progress_info('CLUSTER') AS S
LEFT JOIN pg_database D ON S.datid = D.oid;
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 9b6673867c..cb4fc350c6 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
static List *get_tables_to_cluster(MemoryContext cluster_context);
static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
/*---------------------------------------------------------------------------
@@ -188,16 +188,28 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
/* close relation, keep lock till commit */
table_close(rel, ShareUpdateExclusiveLock);
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
{
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ OidIsValid(indexOid) ? PROGRESS_CLUSTER_COMMAND_CLUSTER :
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
+
/* Do the job. */
cluster_rel(tableOid, indexOid, ¶ms);
+ pgstat_progress_end_command();
}
else
{
List *rvs;
MemoryContext cluster_context;
+ int64 progress_val[2];
+ const int progress_index[] = {
+ PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+ };
+
/* Refuse to hold strong locks in a user transaction */
PreventInTransactionBlock(isTopLevel, "CLUSTER");
@@ -206,7 +218,20 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
ALLOCSET_DEFAULT_SIZES);
rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
- cluster_multiple_rels(rvs, params.options);
+
+ /* Report vacuum full or cluster and number of partitions */
+ progress_val[0] = OidIsValid(indexOid) ?
+ PROGRESS_CLUSTER_COMMAND_CLUSTER :
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL;
+ progress_val[1] = list_length(rvs);
+// This is currently showing the total number of *tables*, including the
+// parent, and intermediate partitions, so the column should be renamed, or
+// we should count the number of leaf partitions
+
+ pgstat_progress_update_multi_param(2, progress_index, progress_val);
+
+ cluster_multiple_rels(rvs, params.options, true);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -245,7 +270,7 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
* cluster_context.
*/
rvs = get_tables_to_cluster(cluster_context);
- cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+ cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -282,14 +307,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* Check for user-requested abort. */
CHECK_FOR_INTERRUPTS();
- pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
- if (OidIsValid(indexOid))
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_CLUSTER);
- else
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
/*
* We grab exclusive access to the target rel and index for the duration
* of the transaction. (This is redundant for the single-transaction
@@ -300,10 +317,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* If the table has gone away, we can skip processing it */
if (!OldHeap)
- {
- pgstat_progress_end_command();
return;
- }
/*
* Since we may open a new transaction for each relation, we have to check
@@ -319,7 +333,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!pg_class_ownercheck(tableOid, GetUserId()))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -334,7 +347,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (RELATION_IS_OTHER_TEMP(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -346,7 +358,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -357,7 +368,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!get_index_isclustered(indexOid))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
}
@@ -416,7 +426,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!RelationIsPopulated(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -424,7 +433,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -440,8 +448,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
rebuild_relation(OldHeap, indexOid, verbose);
/* NB: rebuild_relation does table_close() on OldHeap */
-
- pgstat_progress_end_command();
}
/*
@@ -1366,7 +1372,7 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap,
ReindexParams reindex_params = {0};
int i;
- /* Report that we are now swapping relation files */
+ /* Report that we are now swapping relation files XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_SWAP_REL_FILES);
@@ -1417,13 +1423,13 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap,
else if (newrelpersistence == RELPERSISTENCE_PERMANENT)
reindex_flags |= REINDEX_REL_FORCE_INDEXES_PERMANENT;
- /* Report that we are now reindexing relations */
+ /* Report that we are now reindexing relations XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_REBUILD_INDEX);
reindex_relation(OIDOldHeap, reindex_flags, &reindex_params);
- /* Report that we are now doing clean up */
+ /* Report that we are now doing clean up XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_FINAL_CLEANUP);
@@ -1633,9 +1639,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
/* Cluster each relation in a separate transaction */
static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
{
ListCell *lc;
+ off_t childnum = 0;
/* Commit to get out of starting transaction */
PopActiveSnapshot();
@@ -1658,6 +1665,11 @@ cluster_multiple_rels(List *rvs, int options)
cluster_rel(rvtc->tableOid, rvtc->indexOid,
&cluster_params);
+ if (ispartitioned)
+ pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+ childnum++);
+
+
PopActiveSnapshot();
CommitTransactionCommand();
}
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
#define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS 5
#define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED 6
#define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT 7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL 17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE 18
/* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
#define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP 1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b632d9f2ea..d1fea532bc 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1941,7 +1941,9 @@ pg_stat_progress_cluster| SELECT s.pid,
s.param5 AS heap_tuples_written,
s.param6 AS heap_blks_total,
s.param7 AS heap_blks_scanned,
- s.param8 AS index_rebuild_count
+ s.param8 AS index_rebuild_count,
+ s.param18 AS partitions_total,
+ s.param19 AS partitions_done
FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
LEFT JOIN pg_database d ON ((s.datid = d.oid)));
pg_stat_progress_copy| SELECT s.pid,
--
2.17.0
--O5XBE6gyVG5Rl6Rj
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v8-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"
^ permalink raw reply [nested|flat] 73+ messages in thread
* [PATCH v9 3/8] f! progress reporting..
@ 2021-02-09 02:45 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 73+ messages in thread
From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)
This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.
This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
doc/src/sgml/monitoring.sgml | 21 +++++++++++
src/backend/catalog/system_views.sql | 4 +-
src/backend/commands/cluster.c | 55 ++++++++++++++++------------
src/backend/commands/vacuum.c | 5 +++
src/include/commands/progress.h | 3 ++
src/test/regress/expected/rules.out | 4 +-
6 files changed, 67 insertions(+), 25 deletions(-)
diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml
index 3513e127b7..dca47f6a48 100644
--- a/doc/src/sgml/monitoring.sgml
+++ b/doc/src/sgml/monitoring.sgml
@@ -6231,6 +6231,27 @@ SELECT pg_stat_get_backend_pid(s.backendid) AS pid,
is <literal>rebuilding index</literal>.
</para></entry>
</row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>total_partitions</structfield> <type>bigint</type>
+ </para>
+ When clustering a partitioned table, this column is set to the total
+ number of partitions to be clustered.
+ <para>
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>done_partitions</structfield> <type>bigint</type>
+ </para>
+ <para>
+ When clustering a partitioned table, this column is set to the number
+ of partitions which have been clustered.
+ </para></entry>
+ </row>
+
</tbody>
</tgroup>
</table>
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fc94a73a54..21c27f0437 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1070,7 +1070,9 @@ CREATE VIEW pg_stat_progress_cluster AS
S.param5 AS heap_tuples_written,
S.param6 AS heap_blks_total,
S.param7 AS heap_blks_scanned,
- S.param8 AS index_rebuild_count
+ S.param8 AS index_rebuild_count,
+ S.param18 AS partitions_total,
+ S.param19 AS partitions_done
FROM pg_stat_get_progress_info('CLUSTER') AS S
LEFT JOIN pg_database D ON S.datid = D.oid;
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 0c08ac56dc..7d8d1d8a69 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
static List *get_tables_to_cluster(MemoryContext cluster_context);
static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
/*---------------------------------------------------------------------------
@@ -188,16 +188,26 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
/* close relation, keep lock till commit */
table_close(rel, ShareUpdateExclusiveLock);
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
{
/* Do the job. */
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_CLUSTER);
cluster_rel(tableOid, indexOid, ¶ms);
+ pgstat_progress_end_command();
}
else
{
List *rvs;
MemoryContext cluster_context;
+ int64 progress_val[2];
+ const int progress_index[2] = {
+ PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+ };
+
/* Refuse to hold strong locks in a user transaction */
PreventInTransactionBlock(isTopLevel, "CLUSTER");
@@ -206,7 +216,15 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
ALLOCSET_DEFAULT_SIZES);
rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
- cluster_multiple_rels(rvs, params.options);
+
+ /* Report command and number of partitions */
+ progress_val[0] = PROGRESS_CLUSTER_COMMAND_CLUSTER;
+ /* XXX: rvs includes the parent, which is not a "partition" */
+ progress_val[1] = list_length(rvs);
+
+ pgstat_progress_update_multi_param(2, progress_index, progress_val);
+ cluster_multiple_rels(rvs, params.options, true);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -245,7 +263,11 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
* cluster_context.
*/
rvs = get_tables_to_cluster(cluster_context);
- cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_CLUSTER);
+ cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -282,14 +304,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* Check for user-requested abort. */
CHECK_FOR_INTERRUPTS();
- pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
- if (OidIsValid(indexOid))
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_CLUSTER);
- else
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
/*
* We grab exclusive access to the target rel and index for the duration
* of the transaction. (This is redundant for the single-transaction
@@ -300,10 +314,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* If the table has gone away, we can skip processing it */
if (!OldHeap)
- {
- pgstat_progress_end_command();
return;
- }
/*
* Since we may open a new transaction for each relation, we have to check
@@ -319,7 +330,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!pg_class_ownercheck(tableOid, GetUserId()))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -334,7 +344,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (RELATION_IS_OTHER_TEMP(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -346,7 +355,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -357,7 +365,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!get_index_isclustered(indexOid))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
}
@@ -416,7 +423,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!RelationIsPopulated(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -424,7 +430,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -440,8 +445,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
rebuild_relation(OldHeap, indexOid, verbose);
/* NB: rebuild_relation does table_close() on OldHeap */
-
- pgstat_progress_end_command();
}
/*
@@ -1632,9 +1635,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
/* Cluster each relation in a separate transaction */
static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
{
ListCell *lc;
+ off_t childnum = 0;
/* Commit to get out of starting transaction */
PopActiveSnapshot();
@@ -1657,6 +1661,11 @@ cluster_multiple_rels(List *rvs, int options)
cluster_rel(rvtc->tableOid, rvtc->indexOid,
&cluster_params);
+ if (ispartitioned)
+ pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+ ++childnum);
+
+
PopActiveSnapshot();
CommitTransactionCommand();
}
diff --git a/src/backend/commands/vacuum.c b/src/backend/commands/vacuum.c
index c064352e23..05d52858b1 100644
--- a/src/backend/commands/vacuum.c
+++ b/src/backend/commands/vacuum.c
@@ -37,6 +37,7 @@
#include "catalog/pg_namespace.h"
#include "commands/cluster.h"
#include "commands/defrem.h"
+#include "commands/progress.h"
#include "commands/vacuum.h"
#include "miscadmin.h"
#include "nodes/makefuncs.h"
@@ -1937,7 +1938,11 @@ vacuum_rel(Oid relid, RangeVar *relation, VacuumParams *params)
cluster_params.options |= CLUOPT_VERBOSE;
/* VACUUM FULL is now a variant of CLUSTER; see cluster.c */
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, relid);
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
cluster_rel(relid, InvalidOid, &cluster_params);
+ pgstat_progress_end_command();
}
else
table_relation_vacuum(onerel, params, vac_strategy);
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
#define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS 5
#define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED 6
#define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT 7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL 17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE 18
/* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
#define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP 1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b1c9b7bdfe..c888415235 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1943,7 +1943,9 @@ pg_stat_progress_cluster| SELECT s.pid,
s.param5 AS heap_tuples_written,
s.param6 AS heap_blks_total,
s.param7 AS heap_blks_scanned,
- s.param8 AS index_rebuild_count
+ s.param8 AS index_rebuild_count,
+ s.param18 AS partitions_total,
+ s.param19 AS partitions_done
FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
LEFT JOIN pg_database d ON ((s.datid = d.oid)));
pg_stat_progress_copy| SELECT s.pid,
--
2.17.0
--fmvA4kSBHQVZhkR6
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v9-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"
^ permalink raw reply [nested|flat] 73+ messages in thread
* [PATCH v9 3/8] f! progress reporting..
@ 2021-02-09 02:45 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 73+ messages in thread
From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)
This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.
This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
doc/src/sgml/monitoring.sgml | 21 +++++++++++
src/backend/catalog/system_views.sql | 4 +-
src/backend/commands/cluster.c | 55 ++++++++++++++++------------
src/backend/commands/vacuum.c | 5 +++
src/include/commands/progress.h | 3 ++
src/test/regress/expected/rules.out | 4 +-
6 files changed, 67 insertions(+), 25 deletions(-)
diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml
index 3513e127b7..dca47f6a48 100644
--- a/doc/src/sgml/monitoring.sgml
+++ b/doc/src/sgml/monitoring.sgml
@@ -6231,6 +6231,27 @@ SELECT pg_stat_get_backend_pid(s.backendid) AS pid,
is <literal>rebuilding index</literal>.
</para></entry>
</row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>total_partitions</structfield> <type>bigint</type>
+ </para>
+ When clustering a partitioned table, this column is set to the total
+ number of partitions to be clustered.
+ <para>
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>done_partitions</structfield> <type>bigint</type>
+ </para>
+ <para>
+ When clustering a partitioned table, this column is set to the number
+ of partitions which have been clustered.
+ </para></entry>
+ </row>
+
</tbody>
</tgroup>
</table>
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fc94a73a54..21c27f0437 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1070,7 +1070,9 @@ CREATE VIEW pg_stat_progress_cluster AS
S.param5 AS heap_tuples_written,
S.param6 AS heap_blks_total,
S.param7 AS heap_blks_scanned,
- S.param8 AS index_rebuild_count
+ S.param8 AS index_rebuild_count,
+ S.param18 AS partitions_total,
+ S.param19 AS partitions_done
FROM pg_stat_get_progress_info('CLUSTER') AS S
LEFT JOIN pg_database D ON S.datid = D.oid;
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 0c08ac56dc..7d8d1d8a69 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
static List *get_tables_to_cluster(MemoryContext cluster_context);
static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
/*---------------------------------------------------------------------------
@@ -188,16 +188,26 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
/* close relation, keep lock till commit */
table_close(rel, ShareUpdateExclusiveLock);
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
{
/* Do the job. */
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_CLUSTER);
cluster_rel(tableOid, indexOid, ¶ms);
+ pgstat_progress_end_command();
}
else
{
List *rvs;
MemoryContext cluster_context;
+ int64 progress_val[2];
+ const int progress_index[2] = {
+ PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+ };
+
/* Refuse to hold strong locks in a user transaction */
PreventInTransactionBlock(isTopLevel, "CLUSTER");
@@ -206,7 +216,15 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
ALLOCSET_DEFAULT_SIZES);
rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
- cluster_multiple_rels(rvs, params.options);
+
+ /* Report command and number of partitions */
+ progress_val[0] = PROGRESS_CLUSTER_COMMAND_CLUSTER;
+ /* XXX: rvs includes the parent, which is not a "partition" */
+ progress_val[1] = list_length(rvs);
+
+ pgstat_progress_update_multi_param(2, progress_index, progress_val);
+ cluster_multiple_rels(rvs, params.options, true);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -245,7 +263,11 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
* cluster_context.
*/
rvs = get_tables_to_cluster(cluster_context);
- cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_CLUSTER);
+ cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -282,14 +304,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* Check for user-requested abort. */
CHECK_FOR_INTERRUPTS();
- pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
- if (OidIsValid(indexOid))
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_CLUSTER);
- else
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
/*
* We grab exclusive access to the target rel and index for the duration
* of the transaction. (This is redundant for the single-transaction
@@ -300,10 +314,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* If the table has gone away, we can skip processing it */
if (!OldHeap)
- {
- pgstat_progress_end_command();
return;
- }
/*
* Since we may open a new transaction for each relation, we have to check
@@ -319,7 +330,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!pg_class_ownercheck(tableOid, GetUserId()))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -334,7 +344,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (RELATION_IS_OTHER_TEMP(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -346,7 +355,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -357,7 +365,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!get_index_isclustered(indexOid))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
}
@@ -416,7 +423,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!RelationIsPopulated(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -424,7 +430,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -440,8 +445,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
rebuild_relation(OldHeap, indexOid, verbose);
/* NB: rebuild_relation does table_close() on OldHeap */
-
- pgstat_progress_end_command();
}
/*
@@ -1632,9 +1635,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
/* Cluster each relation in a separate transaction */
static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
{
ListCell *lc;
+ off_t childnum = 0;
/* Commit to get out of starting transaction */
PopActiveSnapshot();
@@ -1657,6 +1661,11 @@ cluster_multiple_rels(List *rvs, int options)
cluster_rel(rvtc->tableOid, rvtc->indexOid,
&cluster_params);
+ if (ispartitioned)
+ pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+ ++childnum);
+
+
PopActiveSnapshot();
CommitTransactionCommand();
}
diff --git a/src/backend/commands/vacuum.c b/src/backend/commands/vacuum.c
index c064352e23..05d52858b1 100644
--- a/src/backend/commands/vacuum.c
+++ b/src/backend/commands/vacuum.c
@@ -37,6 +37,7 @@
#include "catalog/pg_namespace.h"
#include "commands/cluster.h"
#include "commands/defrem.h"
+#include "commands/progress.h"
#include "commands/vacuum.h"
#include "miscadmin.h"
#include "nodes/makefuncs.h"
@@ -1937,7 +1938,11 @@ vacuum_rel(Oid relid, RangeVar *relation, VacuumParams *params)
cluster_params.options |= CLUOPT_VERBOSE;
/* VACUUM FULL is now a variant of CLUSTER; see cluster.c */
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, relid);
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
cluster_rel(relid, InvalidOid, &cluster_params);
+ pgstat_progress_end_command();
}
else
table_relation_vacuum(onerel, params, vac_strategy);
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
#define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS 5
#define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED 6
#define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT 7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL 17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE 18
/* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
#define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP 1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b1c9b7bdfe..c888415235 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1943,7 +1943,9 @@ pg_stat_progress_cluster| SELECT s.pid,
s.param5 AS heap_tuples_written,
s.param6 AS heap_blks_total,
s.param7 AS heap_blks_scanned,
- s.param8 AS index_rebuild_count
+ s.param8 AS index_rebuild_count,
+ s.param18 AS partitions_total,
+ s.param19 AS partitions_done
FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
LEFT JOIN pg_database d ON ((s.datid = d.oid)));
pg_stat_progress_copy| SELECT s.pid,
--
2.17.0
--fmvA4kSBHQVZhkR6
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v9-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"
^ permalink raw reply [nested|flat] 73+ messages in thread
* [PATCH v9 3/8] f! progress reporting..
@ 2021-02-09 02:45 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 73+ messages in thread
From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)
This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.
This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
doc/src/sgml/monitoring.sgml | 21 +++++++++++
src/backend/catalog/system_views.sql | 4 +-
src/backend/commands/cluster.c | 55 ++++++++++++++++------------
src/backend/commands/vacuum.c | 5 +++
src/include/commands/progress.h | 3 ++
src/test/regress/expected/rules.out | 4 +-
6 files changed, 67 insertions(+), 25 deletions(-)
diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml
index 3513e127b7..dca47f6a48 100644
--- a/doc/src/sgml/monitoring.sgml
+++ b/doc/src/sgml/monitoring.sgml
@@ -6231,6 +6231,27 @@ SELECT pg_stat_get_backend_pid(s.backendid) AS pid,
is <literal>rebuilding index</literal>.
</para></entry>
</row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>total_partitions</structfield> <type>bigint</type>
+ </para>
+ When clustering a partitioned table, this column is set to the total
+ number of partitions to be clustered.
+ <para>
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>done_partitions</structfield> <type>bigint</type>
+ </para>
+ <para>
+ When clustering a partitioned table, this column is set to the number
+ of partitions which have been clustered.
+ </para></entry>
+ </row>
+
</tbody>
</tgroup>
</table>
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fc94a73a54..21c27f0437 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1070,7 +1070,9 @@ CREATE VIEW pg_stat_progress_cluster AS
S.param5 AS heap_tuples_written,
S.param6 AS heap_blks_total,
S.param7 AS heap_blks_scanned,
- S.param8 AS index_rebuild_count
+ S.param8 AS index_rebuild_count,
+ S.param18 AS partitions_total,
+ S.param19 AS partitions_done
FROM pg_stat_get_progress_info('CLUSTER') AS S
LEFT JOIN pg_database D ON S.datid = D.oid;
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 0c08ac56dc..7d8d1d8a69 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
static List *get_tables_to_cluster(MemoryContext cluster_context);
static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
/*---------------------------------------------------------------------------
@@ -188,16 +188,26 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
/* close relation, keep lock till commit */
table_close(rel, ShareUpdateExclusiveLock);
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
{
/* Do the job. */
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_CLUSTER);
cluster_rel(tableOid, indexOid, ¶ms);
+ pgstat_progress_end_command();
}
else
{
List *rvs;
MemoryContext cluster_context;
+ int64 progress_val[2];
+ const int progress_index[2] = {
+ PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+ };
+
/* Refuse to hold strong locks in a user transaction */
PreventInTransactionBlock(isTopLevel, "CLUSTER");
@@ -206,7 +216,15 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
ALLOCSET_DEFAULT_SIZES);
rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
- cluster_multiple_rels(rvs, params.options);
+
+ /* Report command and number of partitions */
+ progress_val[0] = PROGRESS_CLUSTER_COMMAND_CLUSTER;
+ /* XXX: rvs includes the parent, which is not a "partition" */
+ progress_val[1] = list_length(rvs);
+
+ pgstat_progress_update_multi_param(2, progress_index, progress_val);
+ cluster_multiple_rels(rvs, params.options, true);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -245,7 +263,11 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
* cluster_context.
*/
rvs = get_tables_to_cluster(cluster_context);
- cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_CLUSTER);
+ cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -282,14 +304,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* Check for user-requested abort. */
CHECK_FOR_INTERRUPTS();
- pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
- if (OidIsValid(indexOid))
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_CLUSTER);
- else
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
/*
* We grab exclusive access to the target rel and index for the duration
* of the transaction. (This is redundant for the single-transaction
@@ -300,10 +314,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* If the table has gone away, we can skip processing it */
if (!OldHeap)
- {
- pgstat_progress_end_command();
return;
- }
/*
* Since we may open a new transaction for each relation, we have to check
@@ -319,7 +330,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!pg_class_ownercheck(tableOid, GetUserId()))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -334,7 +344,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (RELATION_IS_OTHER_TEMP(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -346,7 +355,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -357,7 +365,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!get_index_isclustered(indexOid))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
}
@@ -416,7 +423,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!RelationIsPopulated(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -424,7 +430,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -440,8 +445,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
rebuild_relation(OldHeap, indexOid, verbose);
/* NB: rebuild_relation does table_close() on OldHeap */
-
- pgstat_progress_end_command();
}
/*
@@ -1632,9 +1635,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
/* Cluster each relation in a separate transaction */
static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
{
ListCell *lc;
+ off_t childnum = 0;
/* Commit to get out of starting transaction */
PopActiveSnapshot();
@@ -1657,6 +1661,11 @@ cluster_multiple_rels(List *rvs, int options)
cluster_rel(rvtc->tableOid, rvtc->indexOid,
&cluster_params);
+ if (ispartitioned)
+ pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+ ++childnum);
+
+
PopActiveSnapshot();
CommitTransactionCommand();
}
diff --git a/src/backend/commands/vacuum.c b/src/backend/commands/vacuum.c
index c064352e23..05d52858b1 100644
--- a/src/backend/commands/vacuum.c
+++ b/src/backend/commands/vacuum.c
@@ -37,6 +37,7 @@
#include "catalog/pg_namespace.h"
#include "commands/cluster.h"
#include "commands/defrem.h"
+#include "commands/progress.h"
#include "commands/vacuum.h"
#include "miscadmin.h"
#include "nodes/makefuncs.h"
@@ -1937,7 +1938,11 @@ vacuum_rel(Oid relid, RangeVar *relation, VacuumParams *params)
cluster_params.options |= CLUOPT_VERBOSE;
/* VACUUM FULL is now a variant of CLUSTER; see cluster.c */
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, relid);
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
cluster_rel(relid, InvalidOid, &cluster_params);
+ pgstat_progress_end_command();
}
else
table_relation_vacuum(onerel, params, vac_strategy);
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
#define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS 5
#define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED 6
#define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT 7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL 17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE 18
/* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
#define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP 1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b1c9b7bdfe..c888415235 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1943,7 +1943,9 @@ pg_stat_progress_cluster| SELECT s.pid,
s.param5 AS heap_tuples_written,
s.param6 AS heap_blks_total,
s.param7 AS heap_blks_scanned,
- s.param8 AS index_rebuild_count
+ s.param8 AS index_rebuild_count,
+ s.param18 AS partitions_total,
+ s.param19 AS partitions_done
FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
LEFT JOIN pg_database d ON ((s.datid = d.oid)));
pg_stat_progress_copy| SELECT s.pid,
--
2.17.0
--fmvA4kSBHQVZhkR6
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v9-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"
^ permalink raw reply [nested|flat] 73+ messages in thread
* [PATCH v9 3/8] f! progress reporting..
@ 2021-02-09 02:45 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 73+ messages in thread
From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)
This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.
This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
doc/src/sgml/monitoring.sgml | 21 +++++++++++
src/backend/catalog/system_views.sql | 4 +-
src/backend/commands/cluster.c | 55 ++++++++++++++++------------
src/backend/commands/vacuum.c | 5 +++
src/include/commands/progress.h | 3 ++
src/test/regress/expected/rules.out | 4 +-
6 files changed, 67 insertions(+), 25 deletions(-)
diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml
index 3513e127b7..dca47f6a48 100644
--- a/doc/src/sgml/monitoring.sgml
+++ b/doc/src/sgml/monitoring.sgml
@@ -6231,6 +6231,27 @@ SELECT pg_stat_get_backend_pid(s.backendid) AS pid,
is <literal>rebuilding index</literal>.
</para></entry>
</row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>total_partitions</structfield> <type>bigint</type>
+ </para>
+ When clustering a partitioned table, this column is set to the total
+ number of partitions to be clustered.
+ <para>
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>done_partitions</structfield> <type>bigint</type>
+ </para>
+ <para>
+ When clustering a partitioned table, this column is set to the number
+ of partitions which have been clustered.
+ </para></entry>
+ </row>
+
</tbody>
</tgroup>
</table>
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fc94a73a54..21c27f0437 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1070,7 +1070,9 @@ CREATE VIEW pg_stat_progress_cluster AS
S.param5 AS heap_tuples_written,
S.param6 AS heap_blks_total,
S.param7 AS heap_blks_scanned,
- S.param8 AS index_rebuild_count
+ S.param8 AS index_rebuild_count,
+ S.param18 AS partitions_total,
+ S.param19 AS partitions_done
FROM pg_stat_get_progress_info('CLUSTER') AS S
LEFT JOIN pg_database D ON S.datid = D.oid;
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 0c08ac56dc..7d8d1d8a69 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
static List *get_tables_to_cluster(MemoryContext cluster_context);
static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
/*---------------------------------------------------------------------------
@@ -188,16 +188,26 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
/* close relation, keep lock till commit */
table_close(rel, ShareUpdateExclusiveLock);
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
{
/* Do the job. */
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_CLUSTER);
cluster_rel(tableOid, indexOid, ¶ms);
+ pgstat_progress_end_command();
}
else
{
List *rvs;
MemoryContext cluster_context;
+ int64 progress_val[2];
+ const int progress_index[2] = {
+ PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+ };
+
/* Refuse to hold strong locks in a user transaction */
PreventInTransactionBlock(isTopLevel, "CLUSTER");
@@ -206,7 +216,15 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
ALLOCSET_DEFAULT_SIZES);
rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
- cluster_multiple_rels(rvs, params.options);
+
+ /* Report command and number of partitions */
+ progress_val[0] = PROGRESS_CLUSTER_COMMAND_CLUSTER;
+ /* XXX: rvs includes the parent, which is not a "partition" */
+ progress_val[1] = list_length(rvs);
+
+ pgstat_progress_update_multi_param(2, progress_index, progress_val);
+ cluster_multiple_rels(rvs, params.options, true);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -245,7 +263,11 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
* cluster_context.
*/
rvs = get_tables_to_cluster(cluster_context);
- cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_CLUSTER);
+ cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -282,14 +304,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* Check for user-requested abort. */
CHECK_FOR_INTERRUPTS();
- pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
- if (OidIsValid(indexOid))
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_CLUSTER);
- else
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
/*
* We grab exclusive access to the target rel and index for the duration
* of the transaction. (This is redundant for the single-transaction
@@ -300,10 +314,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* If the table has gone away, we can skip processing it */
if (!OldHeap)
- {
- pgstat_progress_end_command();
return;
- }
/*
* Since we may open a new transaction for each relation, we have to check
@@ -319,7 +330,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!pg_class_ownercheck(tableOid, GetUserId()))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -334,7 +344,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (RELATION_IS_OTHER_TEMP(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -346,7 +355,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -357,7 +365,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!get_index_isclustered(indexOid))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
}
@@ -416,7 +423,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!RelationIsPopulated(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -424,7 +430,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -440,8 +445,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
rebuild_relation(OldHeap, indexOid, verbose);
/* NB: rebuild_relation does table_close() on OldHeap */
-
- pgstat_progress_end_command();
}
/*
@@ -1632,9 +1635,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
/* Cluster each relation in a separate transaction */
static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
{
ListCell *lc;
+ off_t childnum = 0;
/* Commit to get out of starting transaction */
PopActiveSnapshot();
@@ -1657,6 +1661,11 @@ cluster_multiple_rels(List *rvs, int options)
cluster_rel(rvtc->tableOid, rvtc->indexOid,
&cluster_params);
+ if (ispartitioned)
+ pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+ ++childnum);
+
+
PopActiveSnapshot();
CommitTransactionCommand();
}
diff --git a/src/backend/commands/vacuum.c b/src/backend/commands/vacuum.c
index c064352e23..05d52858b1 100644
--- a/src/backend/commands/vacuum.c
+++ b/src/backend/commands/vacuum.c
@@ -37,6 +37,7 @@
#include "catalog/pg_namespace.h"
#include "commands/cluster.h"
#include "commands/defrem.h"
+#include "commands/progress.h"
#include "commands/vacuum.h"
#include "miscadmin.h"
#include "nodes/makefuncs.h"
@@ -1937,7 +1938,11 @@ vacuum_rel(Oid relid, RangeVar *relation, VacuumParams *params)
cluster_params.options |= CLUOPT_VERBOSE;
/* VACUUM FULL is now a variant of CLUSTER; see cluster.c */
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, relid);
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
cluster_rel(relid, InvalidOid, &cluster_params);
+ pgstat_progress_end_command();
}
else
table_relation_vacuum(onerel, params, vac_strategy);
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
#define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS 5
#define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED 6
#define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT 7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL 17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE 18
/* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
#define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP 1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b1c9b7bdfe..c888415235 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1943,7 +1943,9 @@ pg_stat_progress_cluster| SELECT s.pid,
s.param5 AS heap_tuples_written,
s.param6 AS heap_blks_total,
s.param7 AS heap_blks_scanned,
- s.param8 AS index_rebuild_count
+ s.param8 AS index_rebuild_count,
+ s.param18 AS partitions_total,
+ s.param19 AS partitions_done
FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
LEFT JOIN pg_database d ON ((s.datid = d.oid)));
pg_stat_progress_copy| SELECT s.pid,
--
2.17.0
--fmvA4kSBHQVZhkR6
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v9-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"
^ permalink raw reply [nested|flat] 73+ messages in thread
* [PATCH v8 3/8] f! progress reporting..
@ 2021-02-09 02:45 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 73+ messages in thread
From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)
This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.
This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
src/backend/catalog/system_views.sql | 4 +-
src/backend/commands/cluster.c | 64 +++++++++++++++++-----------
src/include/commands/progress.h | 3 ++
src/test/regress/expected/rules.out | 4 +-
4 files changed, 47 insertions(+), 28 deletions(-)
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fa58afd9d7..9a3cb5f935 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1069,7 +1069,9 @@ CREATE VIEW pg_stat_progress_cluster AS
S.param5 AS heap_tuples_written,
S.param6 AS heap_blks_total,
S.param7 AS heap_blks_scanned,
- S.param8 AS index_rebuild_count
+ S.param8 AS index_rebuild_count,
+ S.param18 AS partitions_total,
+ S.param19 AS partitions_done
FROM pg_stat_get_progress_info('CLUSTER') AS S
LEFT JOIN pg_database D ON S.datid = D.oid;
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 9b6673867c..cb4fc350c6 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
static List *get_tables_to_cluster(MemoryContext cluster_context);
static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
/*---------------------------------------------------------------------------
@@ -188,16 +188,28 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
/* close relation, keep lock till commit */
table_close(rel, ShareUpdateExclusiveLock);
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
{
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ OidIsValid(indexOid) ? PROGRESS_CLUSTER_COMMAND_CLUSTER :
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
+
/* Do the job. */
cluster_rel(tableOid, indexOid, ¶ms);
+ pgstat_progress_end_command();
}
else
{
List *rvs;
MemoryContext cluster_context;
+ int64 progress_val[2];
+ const int progress_index[] = {
+ PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+ };
+
/* Refuse to hold strong locks in a user transaction */
PreventInTransactionBlock(isTopLevel, "CLUSTER");
@@ -206,7 +218,20 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
ALLOCSET_DEFAULT_SIZES);
rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
- cluster_multiple_rels(rvs, params.options);
+
+ /* Report vacuum full or cluster and number of partitions */
+ progress_val[0] = OidIsValid(indexOid) ?
+ PROGRESS_CLUSTER_COMMAND_CLUSTER :
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL;
+ progress_val[1] = list_length(rvs);
+// This is currently showing the total number of *tables*, including the
+// parent, and intermediate partitions, so the column should be renamed, or
+// we should count the number of leaf partitions
+
+ pgstat_progress_update_multi_param(2, progress_index, progress_val);
+
+ cluster_multiple_rels(rvs, params.options, true);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -245,7 +270,7 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
* cluster_context.
*/
rvs = get_tables_to_cluster(cluster_context);
- cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+ cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -282,14 +307,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* Check for user-requested abort. */
CHECK_FOR_INTERRUPTS();
- pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
- if (OidIsValid(indexOid))
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_CLUSTER);
- else
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
/*
* We grab exclusive access to the target rel and index for the duration
* of the transaction. (This is redundant for the single-transaction
@@ -300,10 +317,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* If the table has gone away, we can skip processing it */
if (!OldHeap)
- {
- pgstat_progress_end_command();
return;
- }
/*
* Since we may open a new transaction for each relation, we have to check
@@ -319,7 +333,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!pg_class_ownercheck(tableOid, GetUserId()))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -334,7 +347,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (RELATION_IS_OTHER_TEMP(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -346,7 +358,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -357,7 +368,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!get_index_isclustered(indexOid))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
}
@@ -416,7 +426,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!RelationIsPopulated(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -424,7 +433,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -440,8 +448,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
rebuild_relation(OldHeap, indexOid, verbose);
/* NB: rebuild_relation does table_close() on OldHeap */
-
- pgstat_progress_end_command();
}
/*
@@ -1366,7 +1372,7 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap,
ReindexParams reindex_params = {0};
int i;
- /* Report that we are now swapping relation files */
+ /* Report that we are now swapping relation files XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_SWAP_REL_FILES);
@@ -1417,13 +1423,13 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap,
else if (newrelpersistence == RELPERSISTENCE_PERMANENT)
reindex_flags |= REINDEX_REL_FORCE_INDEXES_PERMANENT;
- /* Report that we are now reindexing relations */
+ /* Report that we are now reindexing relations XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_REBUILD_INDEX);
reindex_relation(OIDOldHeap, reindex_flags, &reindex_params);
- /* Report that we are now doing clean up */
+ /* Report that we are now doing clean up XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_FINAL_CLEANUP);
@@ -1633,9 +1639,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
/* Cluster each relation in a separate transaction */
static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
{
ListCell *lc;
+ off_t childnum = 0;
/* Commit to get out of starting transaction */
PopActiveSnapshot();
@@ -1658,6 +1665,11 @@ cluster_multiple_rels(List *rvs, int options)
cluster_rel(rvtc->tableOid, rvtc->indexOid,
&cluster_params);
+ if (ispartitioned)
+ pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+ childnum++);
+
+
PopActiveSnapshot();
CommitTransactionCommand();
}
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
#define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS 5
#define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED 6
#define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT 7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL 17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE 18
/* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
#define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP 1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b632d9f2ea..d1fea532bc 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1941,7 +1941,9 @@ pg_stat_progress_cluster| SELECT s.pid,
s.param5 AS heap_tuples_written,
s.param6 AS heap_blks_total,
s.param7 AS heap_blks_scanned,
- s.param8 AS index_rebuild_count
+ s.param8 AS index_rebuild_count,
+ s.param18 AS partitions_total,
+ s.param19 AS partitions_done
FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
LEFT JOIN pg_database d ON ((s.datid = d.oid)));
pg_stat_progress_copy| SELECT s.pid,
--
2.17.0
--O5XBE6gyVG5Rl6Rj
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v8-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"
^ permalink raw reply [nested|flat] 73+ messages in thread
* [PATCH v9 3/8] f! progress reporting..
@ 2021-02-09 02:45 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 73+ messages in thread
From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)
This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.
This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
doc/src/sgml/monitoring.sgml | 21 +++++++++++
src/backend/catalog/system_views.sql | 4 +-
src/backend/commands/cluster.c | 55 ++++++++++++++++------------
src/backend/commands/vacuum.c | 5 +++
src/include/commands/progress.h | 3 ++
src/test/regress/expected/rules.out | 4 +-
6 files changed, 67 insertions(+), 25 deletions(-)
diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml
index 3513e127b7..dca47f6a48 100644
--- a/doc/src/sgml/monitoring.sgml
+++ b/doc/src/sgml/monitoring.sgml
@@ -6231,6 +6231,27 @@ SELECT pg_stat_get_backend_pid(s.backendid) AS pid,
is <literal>rebuilding index</literal>.
</para></entry>
</row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>total_partitions</structfield> <type>bigint</type>
+ </para>
+ When clustering a partitioned table, this column is set to the total
+ number of partitions to be clustered.
+ <para>
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>done_partitions</structfield> <type>bigint</type>
+ </para>
+ <para>
+ When clustering a partitioned table, this column is set to the number
+ of partitions which have been clustered.
+ </para></entry>
+ </row>
+
</tbody>
</tgroup>
</table>
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fc94a73a54..21c27f0437 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1070,7 +1070,9 @@ CREATE VIEW pg_stat_progress_cluster AS
S.param5 AS heap_tuples_written,
S.param6 AS heap_blks_total,
S.param7 AS heap_blks_scanned,
- S.param8 AS index_rebuild_count
+ S.param8 AS index_rebuild_count,
+ S.param18 AS partitions_total,
+ S.param19 AS partitions_done
FROM pg_stat_get_progress_info('CLUSTER') AS S
LEFT JOIN pg_database D ON S.datid = D.oid;
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 0c08ac56dc..7d8d1d8a69 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
static List *get_tables_to_cluster(MemoryContext cluster_context);
static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
/*---------------------------------------------------------------------------
@@ -188,16 +188,26 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
/* close relation, keep lock till commit */
table_close(rel, ShareUpdateExclusiveLock);
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
{
/* Do the job. */
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_CLUSTER);
cluster_rel(tableOid, indexOid, ¶ms);
+ pgstat_progress_end_command();
}
else
{
List *rvs;
MemoryContext cluster_context;
+ int64 progress_val[2];
+ const int progress_index[2] = {
+ PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+ };
+
/* Refuse to hold strong locks in a user transaction */
PreventInTransactionBlock(isTopLevel, "CLUSTER");
@@ -206,7 +216,15 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
ALLOCSET_DEFAULT_SIZES);
rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
- cluster_multiple_rels(rvs, params.options);
+
+ /* Report command and number of partitions */
+ progress_val[0] = PROGRESS_CLUSTER_COMMAND_CLUSTER;
+ /* XXX: rvs includes the parent, which is not a "partition" */
+ progress_val[1] = list_length(rvs);
+
+ pgstat_progress_update_multi_param(2, progress_index, progress_val);
+ cluster_multiple_rels(rvs, params.options, true);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -245,7 +263,11 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
* cluster_context.
*/
rvs = get_tables_to_cluster(cluster_context);
- cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_CLUSTER);
+ cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -282,14 +304,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* Check for user-requested abort. */
CHECK_FOR_INTERRUPTS();
- pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
- if (OidIsValid(indexOid))
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_CLUSTER);
- else
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
/*
* We grab exclusive access to the target rel and index for the duration
* of the transaction. (This is redundant for the single-transaction
@@ -300,10 +314,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* If the table has gone away, we can skip processing it */
if (!OldHeap)
- {
- pgstat_progress_end_command();
return;
- }
/*
* Since we may open a new transaction for each relation, we have to check
@@ -319,7 +330,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!pg_class_ownercheck(tableOid, GetUserId()))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -334,7 +344,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (RELATION_IS_OTHER_TEMP(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -346,7 +355,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -357,7 +365,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!get_index_isclustered(indexOid))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
}
@@ -416,7 +423,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!RelationIsPopulated(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -424,7 +430,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -440,8 +445,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
rebuild_relation(OldHeap, indexOid, verbose);
/* NB: rebuild_relation does table_close() on OldHeap */
-
- pgstat_progress_end_command();
}
/*
@@ -1632,9 +1635,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
/* Cluster each relation in a separate transaction */
static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
{
ListCell *lc;
+ off_t childnum = 0;
/* Commit to get out of starting transaction */
PopActiveSnapshot();
@@ -1657,6 +1661,11 @@ cluster_multiple_rels(List *rvs, int options)
cluster_rel(rvtc->tableOid, rvtc->indexOid,
&cluster_params);
+ if (ispartitioned)
+ pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+ ++childnum);
+
+
PopActiveSnapshot();
CommitTransactionCommand();
}
diff --git a/src/backend/commands/vacuum.c b/src/backend/commands/vacuum.c
index c064352e23..05d52858b1 100644
--- a/src/backend/commands/vacuum.c
+++ b/src/backend/commands/vacuum.c
@@ -37,6 +37,7 @@
#include "catalog/pg_namespace.h"
#include "commands/cluster.h"
#include "commands/defrem.h"
+#include "commands/progress.h"
#include "commands/vacuum.h"
#include "miscadmin.h"
#include "nodes/makefuncs.h"
@@ -1937,7 +1938,11 @@ vacuum_rel(Oid relid, RangeVar *relation, VacuumParams *params)
cluster_params.options |= CLUOPT_VERBOSE;
/* VACUUM FULL is now a variant of CLUSTER; see cluster.c */
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, relid);
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
cluster_rel(relid, InvalidOid, &cluster_params);
+ pgstat_progress_end_command();
}
else
table_relation_vacuum(onerel, params, vac_strategy);
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
#define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS 5
#define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED 6
#define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT 7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL 17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE 18
/* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
#define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP 1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b1c9b7bdfe..c888415235 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1943,7 +1943,9 @@ pg_stat_progress_cluster| SELECT s.pid,
s.param5 AS heap_tuples_written,
s.param6 AS heap_blks_total,
s.param7 AS heap_blks_scanned,
- s.param8 AS index_rebuild_count
+ s.param8 AS index_rebuild_count,
+ s.param18 AS partitions_total,
+ s.param19 AS partitions_done
FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
LEFT JOIN pg_database d ON ((s.datid = d.oid)));
pg_stat_progress_copy| SELECT s.pid,
--
2.17.0
--fmvA4kSBHQVZhkR6
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v9-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"
^ permalink raw reply [nested|flat] 73+ messages in thread
* [PATCH v9 3/8] f! progress reporting..
@ 2021-02-09 02:45 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 73+ messages in thread
From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)
This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.
This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
doc/src/sgml/monitoring.sgml | 21 +++++++++++
src/backend/catalog/system_views.sql | 4 +-
src/backend/commands/cluster.c | 55 ++++++++++++++++------------
src/backend/commands/vacuum.c | 5 +++
src/include/commands/progress.h | 3 ++
src/test/regress/expected/rules.out | 4 +-
6 files changed, 67 insertions(+), 25 deletions(-)
diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml
index 3513e127b7..dca47f6a48 100644
--- a/doc/src/sgml/monitoring.sgml
+++ b/doc/src/sgml/monitoring.sgml
@@ -6231,6 +6231,27 @@ SELECT pg_stat_get_backend_pid(s.backendid) AS pid,
is <literal>rebuilding index</literal>.
</para></entry>
</row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>total_partitions</structfield> <type>bigint</type>
+ </para>
+ When clustering a partitioned table, this column is set to the total
+ number of partitions to be clustered.
+ <para>
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>done_partitions</structfield> <type>bigint</type>
+ </para>
+ <para>
+ When clustering a partitioned table, this column is set to the number
+ of partitions which have been clustered.
+ </para></entry>
+ </row>
+
</tbody>
</tgroup>
</table>
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fc94a73a54..21c27f0437 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1070,7 +1070,9 @@ CREATE VIEW pg_stat_progress_cluster AS
S.param5 AS heap_tuples_written,
S.param6 AS heap_blks_total,
S.param7 AS heap_blks_scanned,
- S.param8 AS index_rebuild_count
+ S.param8 AS index_rebuild_count,
+ S.param18 AS partitions_total,
+ S.param19 AS partitions_done
FROM pg_stat_get_progress_info('CLUSTER') AS S
LEFT JOIN pg_database D ON S.datid = D.oid;
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 0c08ac56dc..7d8d1d8a69 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
static List *get_tables_to_cluster(MemoryContext cluster_context);
static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
/*---------------------------------------------------------------------------
@@ -188,16 +188,26 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
/* close relation, keep lock till commit */
table_close(rel, ShareUpdateExclusiveLock);
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
{
/* Do the job. */
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_CLUSTER);
cluster_rel(tableOid, indexOid, ¶ms);
+ pgstat_progress_end_command();
}
else
{
List *rvs;
MemoryContext cluster_context;
+ int64 progress_val[2];
+ const int progress_index[2] = {
+ PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+ };
+
/* Refuse to hold strong locks in a user transaction */
PreventInTransactionBlock(isTopLevel, "CLUSTER");
@@ -206,7 +216,15 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
ALLOCSET_DEFAULT_SIZES);
rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
- cluster_multiple_rels(rvs, params.options);
+
+ /* Report command and number of partitions */
+ progress_val[0] = PROGRESS_CLUSTER_COMMAND_CLUSTER;
+ /* XXX: rvs includes the parent, which is not a "partition" */
+ progress_val[1] = list_length(rvs);
+
+ pgstat_progress_update_multi_param(2, progress_index, progress_val);
+ cluster_multiple_rels(rvs, params.options, true);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -245,7 +263,11 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
* cluster_context.
*/
rvs = get_tables_to_cluster(cluster_context);
- cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_CLUSTER);
+ cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -282,14 +304,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* Check for user-requested abort. */
CHECK_FOR_INTERRUPTS();
- pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
- if (OidIsValid(indexOid))
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_CLUSTER);
- else
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
/*
* We grab exclusive access to the target rel and index for the duration
* of the transaction. (This is redundant for the single-transaction
@@ -300,10 +314,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* If the table has gone away, we can skip processing it */
if (!OldHeap)
- {
- pgstat_progress_end_command();
return;
- }
/*
* Since we may open a new transaction for each relation, we have to check
@@ -319,7 +330,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!pg_class_ownercheck(tableOid, GetUserId()))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -334,7 +344,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (RELATION_IS_OTHER_TEMP(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -346,7 +355,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -357,7 +365,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!get_index_isclustered(indexOid))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
}
@@ -416,7 +423,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!RelationIsPopulated(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -424,7 +430,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -440,8 +445,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
rebuild_relation(OldHeap, indexOid, verbose);
/* NB: rebuild_relation does table_close() on OldHeap */
-
- pgstat_progress_end_command();
}
/*
@@ -1632,9 +1635,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
/* Cluster each relation in a separate transaction */
static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
{
ListCell *lc;
+ off_t childnum = 0;
/* Commit to get out of starting transaction */
PopActiveSnapshot();
@@ -1657,6 +1661,11 @@ cluster_multiple_rels(List *rvs, int options)
cluster_rel(rvtc->tableOid, rvtc->indexOid,
&cluster_params);
+ if (ispartitioned)
+ pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+ ++childnum);
+
+
PopActiveSnapshot();
CommitTransactionCommand();
}
diff --git a/src/backend/commands/vacuum.c b/src/backend/commands/vacuum.c
index c064352e23..05d52858b1 100644
--- a/src/backend/commands/vacuum.c
+++ b/src/backend/commands/vacuum.c
@@ -37,6 +37,7 @@
#include "catalog/pg_namespace.h"
#include "commands/cluster.h"
#include "commands/defrem.h"
+#include "commands/progress.h"
#include "commands/vacuum.h"
#include "miscadmin.h"
#include "nodes/makefuncs.h"
@@ -1937,7 +1938,11 @@ vacuum_rel(Oid relid, RangeVar *relation, VacuumParams *params)
cluster_params.options |= CLUOPT_VERBOSE;
/* VACUUM FULL is now a variant of CLUSTER; see cluster.c */
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, relid);
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
cluster_rel(relid, InvalidOid, &cluster_params);
+ pgstat_progress_end_command();
}
else
table_relation_vacuum(onerel, params, vac_strategy);
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
#define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS 5
#define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED 6
#define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT 7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL 17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE 18
/* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
#define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP 1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b1c9b7bdfe..c888415235 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1943,7 +1943,9 @@ pg_stat_progress_cluster| SELECT s.pid,
s.param5 AS heap_tuples_written,
s.param6 AS heap_blks_total,
s.param7 AS heap_blks_scanned,
- s.param8 AS index_rebuild_count
+ s.param8 AS index_rebuild_count,
+ s.param18 AS partitions_total,
+ s.param19 AS partitions_done
FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
LEFT JOIN pg_database d ON ((s.datid = d.oid)));
pg_stat_progress_copy| SELECT s.pid,
--
2.17.0
--fmvA4kSBHQVZhkR6
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v9-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"
^ permalink raw reply [nested|flat] 73+ messages in thread
* [PATCH v9 3/8] f! progress reporting..
@ 2021-02-09 02:45 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 73+ messages in thread
From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)
This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.
This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
doc/src/sgml/monitoring.sgml | 21 +++++++++++
src/backend/catalog/system_views.sql | 4 +-
src/backend/commands/cluster.c | 55 ++++++++++++++++------------
src/backend/commands/vacuum.c | 5 +++
src/include/commands/progress.h | 3 ++
src/test/regress/expected/rules.out | 4 +-
6 files changed, 67 insertions(+), 25 deletions(-)
diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml
index 3513e127b7..dca47f6a48 100644
--- a/doc/src/sgml/monitoring.sgml
+++ b/doc/src/sgml/monitoring.sgml
@@ -6231,6 +6231,27 @@ SELECT pg_stat_get_backend_pid(s.backendid) AS pid,
is <literal>rebuilding index</literal>.
</para></entry>
</row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>total_partitions</structfield> <type>bigint</type>
+ </para>
+ When clustering a partitioned table, this column is set to the total
+ number of partitions to be clustered.
+ <para>
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>done_partitions</structfield> <type>bigint</type>
+ </para>
+ <para>
+ When clustering a partitioned table, this column is set to the number
+ of partitions which have been clustered.
+ </para></entry>
+ </row>
+
</tbody>
</tgroup>
</table>
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fc94a73a54..21c27f0437 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1070,7 +1070,9 @@ CREATE VIEW pg_stat_progress_cluster AS
S.param5 AS heap_tuples_written,
S.param6 AS heap_blks_total,
S.param7 AS heap_blks_scanned,
- S.param8 AS index_rebuild_count
+ S.param8 AS index_rebuild_count,
+ S.param18 AS partitions_total,
+ S.param19 AS partitions_done
FROM pg_stat_get_progress_info('CLUSTER') AS S
LEFT JOIN pg_database D ON S.datid = D.oid;
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 0c08ac56dc..7d8d1d8a69 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
static List *get_tables_to_cluster(MemoryContext cluster_context);
static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
/*---------------------------------------------------------------------------
@@ -188,16 +188,26 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
/* close relation, keep lock till commit */
table_close(rel, ShareUpdateExclusiveLock);
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
{
/* Do the job. */
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_CLUSTER);
cluster_rel(tableOid, indexOid, ¶ms);
+ pgstat_progress_end_command();
}
else
{
List *rvs;
MemoryContext cluster_context;
+ int64 progress_val[2];
+ const int progress_index[2] = {
+ PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+ };
+
/* Refuse to hold strong locks in a user transaction */
PreventInTransactionBlock(isTopLevel, "CLUSTER");
@@ -206,7 +216,15 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
ALLOCSET_DEFAULT_SIZES);
rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
- cluster_multiple_rels(rvs, params.options);
+
+ /* Report command and number of partitions */
+ progress_val[0] = PROGRESS_CLUSTER_COMMAND_CLUSTER;
+ /* XXX: rvs includes the parent, which is not a "partition" */
+ progress_val[1] = list_length(rvs);
+
+ pgstat_progress_update_multi_param(2, progress_index, progress_val);
+ cluster_multiple_rels(rvs, params.options, true);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -245,7 +263,11 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
* cluster_context.
*/
rvs = get_tables_to_cluster(cluster_context);
- cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_CLUSTER);
+ cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -282,14 +304,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* Check for user-requested abort. */
CHECK_FOR_INTERRUPTS();
- pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
- if (OidIsValid(indexOid))
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_CLUSTER);
- else
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
/*
* We grab exclusive access to the target rel and index for the duration
* of the transaction. (This is redundant for the single-transaction
@@ -300,10 +314,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* If the table has gone away, we can skip processing it */
if (!OldHeap)
- {
- pgstat_progress_end_command();
return;
- }
/*
* Since we may open a new transaction for each relation, we have to check
@@ -319,7 +330,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!pg_class_ownercheck(tableOid, GetUserId()))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -334,7 +344,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (RELATION_IS_OTHER_TEMP(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -346,7 +355,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -357,7 +365,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!get_index_isclustered(indexOid))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
}
@@ -416,7 +423,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!RelationIsPopulated(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -424,7 +430,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -440,8 +445,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
rebuild_relation(OldHeap, indexOid, verbose);
/* NB: rebuild_relation does table_close() on OldHeap */
-
- pgstat_progress_end_command();
}
/*
@@ -1632,9 +1635,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
/* Cluster each relation in a separate transaction */
static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
{
ListCell *lc;
+ off_t childnum = 0;
/* Commit to get out of starting transaction */
PopActiveSnapshot();
@@ -1657,6 +1661,11 @@ cluster_multiple_rels(List *rvs, int options)
cluster_rel(rvtc->tableOid, rvtc->indexOid,
&cluster_params);
+ if (ispartitioned)
+ pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+ ++childnum);
+
+
PopActiveSnapshot();
CommitTransactionCommand();
}
diff --git a/src/backend/commands/vacuum.c b/src/backend/commands/vacuum.c
index c064352e23..05d52858b1 100644
--- a/src/backend/commands/vacuum.c
+++ b/src/backend/commands/vacuum.c
@@ -37,6 +37,7 @@
#include "catalog/pg_namespace.h"
#include "commands/cluster.h"
#include "commands/defrem.h"
+#include "commands/progress.h"
#include "commands/vacuum.h"
#include "miscadmin.h"
#include "nodes/makefuncs.h"
@@ -1937,7 +1938,11 @@ vacuum_rel(Oid relid, RangeVar *relation, VacuumParams *params)
cluster_params.options |= CLUOPT_VERBOSE;
/* VACUUM FULL is now a variant of CLUSTER; see cluster.c */
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, relid);
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
cluster_rel(relid, InvalidOid, &cluster_params);
+ pgstat_progress_end_command();
}
else
table_relation_vacuum(onerel, params, vac_strategy);
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
#define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS 5
#define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED 6
#define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT 7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL 17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE 18
/* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
#define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP 1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b1c9b7bdfe..c888415235 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1943,7 +1943,9 @@ pg_stat_progress_cluster| SELECT s.pid,
s.param5 AS heap_tuples_written,
s.param6 AS heap_blks_total,
s.param7 AS heap_blks_scanned,
- s.param8 AS index_rebuild_count
+ s.param8 AS index_rebuild_count,
+ s.param18 AS partitions_total,
+ s.param19 AS partitions_done
FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
LEFT JOIN pg_database d ON ((s.datid = d.oid)));
pg_stat_progress_copy| SELECT s.pid,
--
2.17.0
--fmvA4kSBHQVZhkR6
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v9-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"
^ permalink raw reply [nested|flat] 73+ messages in thread
* [PATCH v8 3/8] f! progress reporting..
@ 2021-02-09 02:45 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 73+ messages in thread
From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)
This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.
This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
src/backend/catalog/system_views.sql | 4 +-
src/backend/commands/cluster.c | 64 +++++++++++++++++-----------
src/include/commands/progress.h | 3 ++
src/test/regress/expected/rules.out | 4 +-
4 files changed, 47 insertions(+), 28 deletions(-)
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fa58afd9d7..9a3cb5f935 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1069,7 +1069,9 @@ CREATE VIEW pg_stat_progress_cluster AS
S.param5 AS heap_tuples_written,
S.param6 AS heap_blks_total,
S.param7 AS heap_blks_scanned,
- S.param8 AS index_rebuild_count
+ S.param8 AS index_rebuild_count,
+ S.param18 AS partitions_total,
+ S.param19 AS partitions_done
FROM pg_stat_get_progress_info('CLUSTER') AS S
LEFT JOIN pg_database D ON S.datid = D.oid;
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 9b6673867c..cb4fc350c6 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
static List *get_tables_to_cluster(MemoryContext cluster_context);
static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
/*---------------------------------------------------------------------------
@@ -188,16 +188,28 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
/* close relation, keep lock till commit */
table_close(rel, ShareUpdateExclusiveLock);
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
{
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ OidIsValid(indexOid) ? PROGRESS_CLUSTER_COMMAND_CLUSTER :
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
+
/* Do the job. */
cluster_rel(tableOid, indexOid, ¶ms);
+ pgstat_progress_end_command();
}
else
{
List *rvs;
MemoryContext cluster_context;
+ int64 progress_val[2];
+ const int progress_index[] = {
+ PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+ };
+
/* Refuse to hold strong locks in a user transaction */
PreventInTransactionBlock(isTopLevel, "CLUSTER");
@@ -206,7 +218,20 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
ALLOCSET_DEFAULT_SIZES);
rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
- cluster_multiple_rels(rvs, params.options);
+
+ /* Report vacuum full or cluster and number of partitions */
+ progress_val[0] = OidIsValid(indexOid) ?
+ PROGRESS_CLUSTER_COMMAND_CLUSTER :
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL;
+ progress_val[1] = list_length(rvs);
+// This is currently showing the total number of *tables*, including the
+// parent, and intermediate partitions, so the column should be renamed, or
+// we should count the number of leaf partitions
+
+ pgstat_progress_update_multi_param(2, progress_index, progress_val);
+
+ cluster_multiple_rels(rvs, params.options, true);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -245,7 +270,7 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
* cluster_context.
*/
rvs = get_tables_to_cluster(cluster_context);
- cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+ cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -282,14 +307,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* Check for user-requested abort. */
CHECK_FOR_INTERRUPTS();
- pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
- if (OidIsValid(indexOid))
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_CLUSTER);
- else
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
/*
* We grab exclusive access to the target rel and index for the duration
* of the transaction. (This is redundant for the single-transaction
@@ -300,10 +317,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* If the table has gone away, we can skip processing it */
if (!OldHeap)
- {
- pgstat_progress_end_command();
return;
- }
/*
* Since we may open a new transaction for each relation, we have to check
@@ -319,7 +333,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!pg_class_ownercheck(tableOid, GetUserId()))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -334,7 +347,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (RELATION_IS_OTHER_TEMP(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -346,7 +358,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -357,7 +368,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!get_index_isclustered(indexOid))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
}
@@ -416,7 +426,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!RelationIsPopulated(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -424,7 +433,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -440,8 +448,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
rebuild_relation(OldHeap, indexOid, verbose);
/* NB: rebuild_relation does table_close() on OldHeap */
-
- pgstat_progress_end_command();
}
/*
@@ -1366,7 +1372,7 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap,
ReindexParams reindex_params = {0};
int i;
- /* Report that we are now swapping relation files */
+ /* Report that we are now swapping relation files XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_SWAP_REL_FILES);
@@ -1417,13 +1423,13 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap,
else if (newrelpersistence == RELPERSISTENCE_PERMANENT)
reindex_flags |= REINDEX_REL_FORCE_INDEXES_PERMANENT;
- /* Report that we are now reindexing relations */
+ /* Report that we are now reindexing relations XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_REBUILD_INDEX);
reindex_relation(OIDOldHeap, reindex_flags, &reindex_params);
- /* Report that we are now doing clean up */
+ /* Report that we are now doing clean up XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_FINAL_CLEANUP);
@@ -1633,9 +1639,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
/* Cluster each relation in a separate transaction */
static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
{
ListCell *lc;
+ off_t childnum = 0;
/* Commit to get out of starting transaction */
PopActiveSnapshot();
@@ -1658,6 +1665,11 @@ cluster_multiple_rels(List *rvs, int options)
cluster_rel(rvtc->tableOid, rvtc->indexOid,
&cluster_params);
+ if (ispartitioned)
+ pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+ childnum++);
+
+
PopActiveSnapshot();
CommitTransactionCommand();
}
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
#define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS 5
#define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED 6
#define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT 7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL 17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE 18
/* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
#define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP 1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b632d9f2ea..d1fea532bc 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1941,7 +1941,9 @@ pg_stat_progress_cluster| SELECT s.pid,
s.param5 AS heap_tuples_written,
s.param6 AS heap_blks_total,
s.param7 AS heap_blks_scanned,
- s.param8 AS index_rebuild_count
+ s.param8 AS index_rebuild_count,
+ s.param18 AS partitions_total,
+ s.param19 AS partitions_done
FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
LEFT JOIN pg_database d ON ((s.datid = d.oid)));
pg_stat_progress_copy| SELECT s.pid,
--
2.17.0
--O5XBE6gyVG5Rl6Rj
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v8-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"
^ permalink raw reply [nested|flat] 73+ messages in thread
* [PATCH v8 3/8] f! progress reporting..
@ 2021-02-09 02:45 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 73+ messages in thread
From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)
This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.
This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
src/backend/catalog/system_views.sql | 4 +-
src/backend/commands/cluster.c | 64 +++++++++++++++++-----------
src/include/commands/progress.h | 3 ++
src/test/regress/expected/rules.out | 4 +-
4 files changed, 47 insertions(+), 28 deletions(-)
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fa58afd9d7..9a3cb5f935 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1069,7 +1069,9 @@ CREATE VIEW pg_stat_progress_cluster AS
S.param5 AS heap_tuples_written,
S.param6 AS heap_blks_total,
S.param7 AS heap_blks_scanned,
- S.param8 AS index_rebuild_count
+ S.param8 AS index_rebuild_count,
+ S.param18 AS partitions_total,
+ S.param19 AS partitions_done
FROM pg_stat_get_progress_info('CLUSTER') AS S
LEFT JOIN pg_database D ON S.datid = D.oid;
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 9b6673867c..cb4fc350c6 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
static List *get_tables_to_cluster(MemoryContext cluster_context);
static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
/*---------------------------------------------------------------------------
@@ -188,16 +188,28 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
/* close relation, keep lock till commit */
table_close(rel, ShareUpdateExclusiveLock);
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
{
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ OidIsValid(indexOid) ? PROGRESS_CLUSTER_COMMAND_CLUSTER :
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
+
/* Do the job. */
cluster_rel(tableOid, indexOid, ¶ms);
+ pgstat_progress_end_command();
}
else
{
List *rvs;
MemoryContext cluster_context;
+ int64 progress_val[2];
+ const int progress_index[] = {
+ PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+ };
+
/* Refuse to hold strong locks in a user transaction */
PreventInTransactionBlock(isTopLevel, "CLUSTER");
@@ -206,7 +218,20 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
ALLOCSET_DEFAULT_SIZES);
rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
- cluster_multiple_rels(rvs, params.options);
+
+ /* Report vacuum full or cluster and number of partitions */
+ progress_val[0] = OidIsValid(indexOid) ?
+ PROGRESS_CLUSTER_COMMAND_CLUSTER :
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL;
+ progress_val[1] = list_length(rvs);
+// This is currently showing the total number of *tables*, including the
+// parent, and intermediate partitions, so the column should be renamed, or
+// we should count the number of leaf partitions
+
+ pgstat_progress_update_multi_param(2, progress_index, progress_val);
+
+ cluster_multiple_rels(rvs, params.options, true);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -245,7 +270,7 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
* cluster_context.
*/
rvs = get_tables_to_cluster(cluster_context);
- cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+ cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -282,14 +307,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* Check for user-requested abort. */
CHECK_FOR_INTERRUPTS();
- pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
- if (OidIsValid(indexOid))
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_CLUSTER);
- else
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
/*
* We grab exclusive access to the target rel and index for the duration
* of the transaction. (This is redundant for the single-transaction
@@ -300,10 +317,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* If the table has gone away, we can skip processing it */
if (!OldHeap)
- {
- pgstat_progress_end_command();
return;
- }
/*
* Since we may open a new transaction for each relation, we have to check
@@ -319,7 +333,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!pg_class_ownercheck(tableOid, GetUserId()))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -334,7 +347,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (RELATION_IS_OTHER_TEMP(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -346,7 +358,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -357,7 +368,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!get_index_isclustered(indexOid))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
}
@@ -416,7 +426,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!RelationIsPopulated(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -424,7 +433,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -440,8 +448,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
rebuild_relation(OldHeap, indexOid, verbose);
/* NB: rebuild_relation does table_close() on OldHeap */
-
- pgstat_progress_end_command();
}
/*
@@ -1366,7 +1372,7 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap,
ReindexParams reindex_params = {0};
int i;
- /* Report that we are now swapping relation files */
+ /* Report that we are now swapping relation files XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_SWAP_REL_FILES);
@@ -1417,13 +1423,13 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap,
else if (newrelpersistence == RELPERSISTENCE_PERMANENT)
reindex_flags |= REINDEX_REL_FORCE_INDEXES_PERMANENT;
- /* Report that we are now reindexing relations */
+ /* Report that we are now reindexing relations XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_REBUILD_INDEX);
reindex_relation(OIDOldHeap, reindex_flags, &reindex_params);
- /* Report that we are now doing clean up */
+ /* Report that we are now doing clean up XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_FINAL_CLEANUP);
@@ -1633,9 +1639,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
/* Cluster each relation in a separate transaction */
static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
{
ListCell *lc;
+ off_t childnum = 0;
/* Commit to get out of starting transaction */
PopActiveSnapshot();
@@ -1658,6 +1665,11 @@ cluster_multiple_rels(List *rvs, int options)
cluster_rel(rvtc->tableOid, rvtc->indexOid,
&cluster_params);
+ if (ispartitioned)
+ pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+ childnum++);
+
+
PopActiveSnapshot();
CommitTransactionCommand();
}
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
#define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS 5
#define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED 6
#define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT 7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL 17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE 18
/* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
#define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP 1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b632d9f2ea..d1fea532bc 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1941,7 +1941,9 @@ pg_stat_progress_cluster| SELECT s.pid,
s.param5 AS heap_tuples_written,
s.param6 AS heap_blks_total,
s.param7 AS heap_blks_scanned,
- s.param8 AS index_rebuild_count
+ s.param8 AS index_rebuild_count,
+ s.param18 AS partitions_total,
+ s.param19 AS partitions_done
FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
LEFT JOIN pg_database d ON ((s.datid = d.oid)));
pg_stat_progress_copy| SELECT s.pid,
--
2.17.0
--O5XBE6gyVG5Rl6Rj
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v8-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"
^ permalink raw reply [nested|flat] 73+ messages in thread
* [PATCH v8 3/8] f! progress reporting..
@ 2021-02-09 02:45 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 73+ messages in thread
From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)
This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.
This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
src/backend/catalog/system_views.sql | 4 +-
src/backend/commands/cluster.c | 64 +++++++++++++++++-----------
src/include/commands/progress.h | 3 ++
src/test/regress/expected/rules.out | 4 +-
4 files changed, 47 insertions(+), 28 deletions(-)
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fa58afd9d7..9a3cb5f935 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1069,7 +1069,9 @@ CREATE VIEW pg_stat_progress_cluster AS
S.param5 AS heap_tuples_written,
S.param6 AS heap_blks_total,
S.param7 AS heap_blks_scanned,
- S.param8 AS index_rebuild_count
+ S.param8 AS index_rebuild_count,
+ S.param18 AS partitions_total,
+ S.param19 AS partitions_done
FROM pg_stat_get_progress_info('CLUSTER') AS S
LEFT JOIN pg_database D ON S.datid = D.oid;
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 9b6673867c..cb4fc350c6 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
static List *get_tables_to_cluster(MemoryContext cluster_context);
static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
/*---------------------------------------------------------------------------
@@ -188,16 +188,28 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
/* close relation, keep lock till commit */
table_close(rel, ShareUpdateExclusiveLock);
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
{
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ OidIsValid(indexOid) ? PROGRESS_CLUSTER_COMMAND_CLUSTER :
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
+
/* Do the job. */
cluster_rel(tableOid, indexOid, ¶ms);
+ pgstat_progress_end_command();
}
else
{
List *rvs;
MemoryContext cluster_context;
+ int64 progress_val[2];
+ const int progress_index[] = {
+ PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+ };
+
/* Refuse to hold strong locks in a user transaction */
PreventInTransactionBlock(isTopLevel, "CLUSTER");
@@ -206,7 +218,20 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
ALLOCSET_DEFAULT_SIZES);
rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
- cluster_multiple_rels(rvs, params.options);
+
+ /* Report vacuum full or cluster and number of partitions */
+ progress_val[0] = OidIsValid(indexOid) ?
+ PROGRESS_CLUSTER_COMMAND_CLUSTER :
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL;
+ progress_val[1] = list_length(rvs);
+// This is currently showing the total number of *tables*, including the
+// parent, and intermediate partitions, so the column should be renamed, or
+// we should count the number of leaf partitions
+
+ pgstat_progress_update_multi_param(2, progress_index, progress_val);
+
+ cluster_multiple_rels(rvs, params.options, true);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -245,7 +270,7 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
* cluster_context.
*/
rvs = get_tables_to_cluster(cluster_context);
- cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+ cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -282,14 +307,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* Check for user-requested abort. */
CHECK_FOR_INTERRUPTS();
- pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
- if (OidIsValid(indexOid))
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_CLUSTER);
- else
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
/*
* We grab exclusive access to the target rel and index for the duration
* of the transaction. (This is redundant for the single-transaction
@@ -300,10 +317,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* If the table has gone away, we can skip processing it */
if (!OldHeap)
- {
- pgstat_progress_end_command();
return;
- }
/*
* Since we may open a new transaction for each relation, we have to check
@@ -319,7 +333,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!pg_class_ownercheck(tableOid, GetUserId()))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -334,7 +347,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (RELATION_IS_OTHER_TEMP(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -346,7 +358,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -357,7 +368,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!get_index_isclustered(indexOid))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
}
@@ -416,7 +426,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!RelationIsPopulated(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -424,7 +433,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -440,8 +448,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
rebuild_relation(OldHeap, indexOid, verbose);
/* NB: rebuild_relation does table_close() on OldHeap */
-
- pgstat_progress_end_command();
}
/*
@@ -1366,7 +1372,7 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap,
ReindexParams reindex_params = {0};
int i;
- /* Report that we are now swapping relation files */
+ /* Report that we are now swapping relation files XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_SWAP_REL_FILES);
@@ -1417,13 +1423,13 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap,
else if (newrelpersistence == RELPERSISTENCE_PERMANENT)
reindex_flags |= REINDEX_REL_FORCE_INDEXES_PERMANENT;
- /* Report that we are now reindexing relations */
+ /* Report that we are now reindexing relations XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_REBUILD_INDEX);
reindex_relation(OIDOldHeap, reindex_flags, &reindex_params);
- /* Report that we are now doing clean up */
+ /* Report that we are now doing clean up XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_FINAL_CLEANUP);
@@ -1633,9 +1639,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
/* Cluster each relation in a separate transaction */
static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
{
ListCell *lc;
+ off_t childnum = 0;
/* Commit to get out of starting transaction */
PopActiveSnapshot();
@@ -1658,6 +1665,11 @@ cluster_multiple_rels(List *rvs, int options)
cluster_rel(rvtc->tableOid, rvtc->indexOid,
&cluster_params);
+ if (ispartitioned)
+ pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+ childnum++);
+
+
PopActiveSnapshot();
CommitTransactionCommand();
}
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
#define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS 5
#define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED 6
#define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT 7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL 17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE 18
/* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
#define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP 1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b632d9f2ea..d1fea532bc 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1941,7 +1941,9 @@ pg_stat_progress_cluster| SELECT s.pid,
s.param5 AS heap_tuples_written,
s.param6 AS heap_blks_total,
s.param7 AS heap_blks_scanned,
- s.param8 AS index_rebuild_count
+ s.param8 AS index_rebuild_count,
+ s.param18 AS partitions_total,
+ s.param19 AS partitions_done
FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
LEFT JOIN pg_database d ON ((s.datid = d.oid)));
pg_stat_progress_copy| SELECT s.pid,
--
2.17.0
--O5XBE6gyVG5Rl6Rj
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v8-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"
^ permalink raw reply [nested|flat] 73+ messages in thread
* [PATCH v9 3/8] f! progress reporting..
@ 2021-02-09 02:45 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 73+ messages in thread
From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)
This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.
This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
doc/src/sgml/monitoring.sgml | 21 +++++++++++
src/backend/catalog/system_views.sql | 4 +-
src/backend/commands/cluster.c | 55 ++++++++++++++++------------
src/backend/commands/vacuum.c | 5 +++
src/include/commands/progress.h | 3 ++
src/test/regress/expected/rules.out | 4 +-
6 files changed, 67 insertions(+), 25 deletions(-)
diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml
index 3513e127b7..dca47f6a48 100644
--- a/doc/src/sgml/monitoring.sgml
+++ b/doc/src/sgml/monitoring.sgml
@@ -6231,6 +6231,27 @@ SELECT pg_stat_get_backend_pid(s.backendid) AS pid,
is <literal>rebuilding index</literal>.
</para></entry>
</row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>total_partitions</structfield> <type>bigint</type>
+ </para>
+ When clustering a partitioned table, this column is set to the total
+ number of partitions to be clustered.
+ <para>
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>done_partitions</structfield> <type>bigint</type>
+ </para>
+ <para>
+ When clustering a partitioned table, this column is set to the number
+ of partitions which have been clustered.
+ </para></entry>
+ </row>
+
</tbody>
</tgroup>
</table>
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fc94a73a54..21c27f0437 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1070,7 +1070,9 @@ CREATE VIEW pg_stat_progress_cluster AS
S.param5 AS heap_tuples_written,
S.param6 AS heap_blks_total,
S.param7 AS heap_blks_scanned,
- S.param8 AS index_rebuild_count
+ S.param8 AS index_rebuild_count,
+ S.param18 AS partitions_total,
+ S.param19 AS partitions_done
FROM pg_stat_get_progress_info('CLUSTER') AS S
LEFT JOIN pg_database D ON S.datid = D.oid;
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 0c08ac56dc..7d8d1d8a69 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
static List *get_tables_to_cluster(MemoryContext cluster_context);
static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
/*---------------------------------------------------------------------------
@@ -188,16 +188,26 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
/* close relation, keep lock till commit */
table_close(rel, ShareUpdateExclusiveLock);
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
{
/* Do the job. */
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_CLUSTER);
cluster_rel(tableOid, indexOid, ¶ms);
+ pgstat_progress_end_command();
}
else
{
List *rvs;
MemoryContext cluster_context;
+ int64 progress_val[2];
+ const int progress_index[2] = {
+ PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+ };
+
/* Refuse to hold strong locks in a user transaction */
PreventInTransactionBlock(isTopLevel, "CLUSTER");
@@ -206,7 +216,15 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
ALLOCSET_DEFAULT_SIZES);
rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
- cluster_multiple_rels(rvs, params.options);
+
+ /* Report command and number of partitions */
+ progress_val[0] = PROGRESS_CLUSTER_COMMAND_CLUSTER;
+ /* XXX: rvs includes the parent, which is not a "partition" */
+ progress_val[1] = list_length(rvs);
+
+ pgstat_progress_update_multi_param(2, progress_index, progress_val);
+ cluster_multiple_rels(rvs, params.options, true);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -245,7 +263,11 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
* cluster_context.
*/
rvs = get_tables_to_cluster(cluster_context);
- cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_CLUSTER);
+ cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -282,14 +304,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* Check for user-requested abort. */
CHECK_FOR_INTERRUPTS();
- pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
- if (OidIsValid(indexOid))
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_CLUSTER);
- else
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
/*
* We grab exclusive access to the target rel and index for the duration
* of the transaction. (This is redundant for the single-transaction
@@ -300,10 +314,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* If the table has gone away, we can skip processing it */
if (!OldHeap)
- {
- pgstat_progress_end_command();
return;
- }
/*
* Since we may open a new transaction for each relation, we have to check
@@ -319,7 +330,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!pg_class_ownercheck(tableOid, GetUserId()))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -334,7 +344,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (RELATION_IS_OTHER_TEMP(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -346,7 +355,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -357,7 +365,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!get_index_isclustered(indexOid))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
}
@@ -416,7 +423,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!RelationIsPopulated(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -424,7 +430,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -440,8 +445,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
rebuild_relation(OldHeap, indexOid, verbose);
/* NB: rebuild_relation does table_close() on OldHeap */
-
- pgstat_progress_end_command();
}
/*
@@ -1632,9 +1635,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
/* Cluster each relation in a separate transaction */
static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
{
ListCell *lc;
+ off_t childnum = 0;
/* Commit to get out of starting transaction */
PopActiveSnapshot();
@@ -1657,6 +1661,11 @@ cluster_multiple_rels(List *rvs, int options)
cluster_rel(rvtc->tableOid, rvtc->indexOid,
&cluster_params);
+ if (ispartitioned)
+ pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+ ++childnum);
+
+
PopActiveSnapshot();
CommitTransactionCommand();
}
diff --git a/src/backend/commands/vacuum.c b/src/backend/commands/vacuum.c
index c064352e23..05d52858b1 100644
--- a/src/backend/commands/vacuum.c
+++ b/src/backend/commands/vacuum.c
@@ -37,6 +37,7 @@
#include "catalog/pg_namespace.h"
#include "commands/cluster.h"
#include "commands/defrem.h"
+#include "commands/progress.h"
#include "commands/vacuum.h"
#include "miscadmin.h"
#include "nodes/makefuncs.h"
@@ -1937,7 +1938,11 @@ vacuum_rel(Oid relid, RangeVar *relation, VacuumParams *params)
cluster_params.options |= CLUOPT_VERBOSE;
/* VACUUM FULL is now a variant of CLUSTER; see cluster.c */
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, relid);
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
cluster_rel(relid, InvalidOid, &cluster_params);
+ pgstat_progress_end_command();
}
else
table_relation_vacuum(onerel, params, vac_strategy);
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
#define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS 5
#define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED 6
#define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT 7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL 17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE 18
/* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
#define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP 1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b1c9b7bdfe..c888415235 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1943,7 +1943,9 @@ pg_stat_progress_cluster| SELECT s.pid,
s.param5 AS heap_tuples_written,
s.param6 AS heap_blks_total,
s.param7 AS heap_blks_scanned,
- s.param8 AS index_rebuild_count
+ s.param8 AS index_rebuild_count,
+ s.param18 AS partitions_total,
+ s.param19 AS partitions_done
FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
LEFT JOIN pg_database d ON ((s.datid = d.oid)));
pg_stat_progress_copy| SELECT s.pid,
--
2.17.0
--fmvA4kSBHQVZhkR6
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v9-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"
^ permalink raw reply [nested|flat] 73+ messages in thread
* [PATCH v9 3/8] f! progress reporting..
@ 2021-02-09 02:45 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 73+ messages in thread
From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)
This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.
This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
doc/src/sgml/monitoring.sgml | 21 +++++++++++
src/backend/catalog/system_views.sql | 4 +-
src/backend/commands/cluster.c | 55 ++++++++++++++++------------
src/backend/commands/vacuum.c | 5 +++
src/include/commands/progress.h | 3 ++
src/test/regress/expected/rules.out | 4 +-
6 files changed, 67 insertions(+), 25 deletions(-)
diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml
index 3513e127b7..dca47f6a48 100644
--- a/doc/src/sgml/monitoring.sgml
+++ b/doc/src/sgml/monitoring.sgml
@@ -6231,6 +6231,27 @@ SELECT pg_stat_get_backend_pid(s.backendid) AS pid,
is <literal>rebuilding index</literal>.
</para></entry>
</row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>total_partitions</structfield> <type>bigint</type>
+ </para>
+ When clustering a partitioned table, this column is set to the total
+ number of partitions to be clustered.
+ <para>
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>done_partitions</structfield> <type>bigint</type>
+ </para>
+ <para>
+ When clustering a partitioned table, this column is set to the number
+ of partitions which have been clustered.
+ </para></entry>
+ </row>
+
</tbody>
</tgroup>
</table>
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fc94a73a54..21c27f0437 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1070,7 +1070,9 @@ CREATE VIEW pg_stat_progress_cluster AS
S.param5 AS heap_tuples_written,
S.param6 AS heap_blks_total,
S.param7 AS heap_blks_scanned,
- S.param8 AS index_rebuild_count
+ S.param8 AS index_rebuild_count,
+ S.param18 AS partitions_total,
+ S.param19 AS partitions_done
FROM pg_stat_get_progress_info('CLUSTER') AS S
LEFT JOIN pg_database D ON S.datid = D.oid;
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 0c08ac56dc..7d8d1d8a69 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
static List *get_tables_to_cluster(MemoryContext cluster_context);
static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
/*---------------------------------------------------------------------------
@@ -188,16 +188,26 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
/* close relation, keep lock till commit */
table_close(rel, ShareUpdateExclusiveLock);
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
{
/* Do the job. */
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_CLUSTER);
cluster_rel(tableOid, indexOid, ¶ms);
+ pgstat_progress_end_command();
}
else
{
List *rvs;
MemoryContext cluster_context;
+ int64 progress_val[2];
+ const int progress_index[2] = {
+ PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+ };
+
/* Refuse to hold strong locks in a user transaction */
PreventInTransactionBlock(isTopLevel, "CLUSTER");
@@ -206,7 +216,15 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
ALLOCSET_DEFAULT_SIZES);
rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
- cluster_multiple_rels(rvs, params.options);
+
+ /* Report command and number of partitions */
+ progress_val[0] = PROGRESS_CLUSTER_COMMAND_CLUSTER;
+ /* XXX: rvs includes the parent, which is not a "partition" */
+ progress_val[1] = list_length(rvs);
+
+ pgstat_progress_update_multi_param(2, progress_index, progress_val);
+ cluster_multiple_rels(rvs, params.options, true);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -245,7 +263,11 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
* cluster_context.
*/
rvs = get_tables_to_cluster(cluster_context);
- cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_CLUSTER);
+ cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -282,14 +304,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* Check for user-requested abort. */
CHECK_FOR_INTERRUPTS();
- pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
- if (OidIsValid(indexOid))
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_CLUSTER);
- else
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
/*
* We grab exclusive access to the target rel and index for the duration
* of the transaction. (This is redundant for the single-transaction
@@ -300,10 +314,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* If the table has gone away, we can skip processing it */
if (!OldHeap)
- {
- pgstat_progress_end_command();
return;
- }
/*
* Since we may open a new transaction for each relation, we have to check
@@ -319,7 +330,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!pg_class_ownercheck(tableOid, GetUserId()))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -334,7 +344,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (RELATION_IS_OTHER_TEMP(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -346,7 +355,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -357,7 +365,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!get_index_isclustered(indexOid))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
}
@@ -416,7 +423,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!RelationIsPopulated(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -424,7 +430,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -440,8 +445,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
rebuild_relation(OldHeap, indexOid, verbose);
/* NB: rebuild_relation does table_close() on OldHeap */
-
- pgstat_progress_end_command();
}
/*
@@ -1632,9 +1635,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
/* Cluster each relation in a separate transaction */
static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
{
ListCell *lc;
+ off_t childnum = 0;
/* Commit to get out of starting transaction */
PopActiveSnapshot();
@@ -1657,6 +1661,11 @@ cluster_multiple_rels(List *rvs, int options)
cluster_rel(rvtc->tableOid, rvtc->indexOid,
&cluster_params);
+ if (ispartitioned)
+ pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+ ++childnum);
+
+
PopActiveSnapshot();
CommitTransactionCommand();
}
diff --git a/src/backend/commands/vacuum.c b/src/backend/commands/vacuum.c
index c064352e23..05d52858b1 100644
--- a/src/backend/commands/vacuum.c
+++ b/src/backend/commands/vacuum.c
@@ -37,6 +37,7 @@
#include "catalog/pg_namespace.h"
#include "commands/cluster.h"
#include "commands/defrem.h"
+#include "commands/progress.h"
#include "commands/vacuum.h"
#include "miscadmin.h"
#include "nodes/makefuncs.h"
@@ -1937,7 +1938,11 @@ vacuum_rel(Oid relid, RangeVar *relation, VacuumParams *params)
cluster_params.options |= CLUOPT_VERBOSE;
/* VACUUM FULL is now a variant of CLUSTER; see cluster.c */
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, relid);
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
cluster_rel(relid, InvalidOid, &cluster_params);
+ pgstat_progress_end_command();
}
else
table_relation_vacuum(onerel, params, vac_strategy);
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
#define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS 5
#define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED 6
#define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT 7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL 17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE 18
/* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
#define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP 1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b1c9b7bdfe..c888415235 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1943,7 +1943,9 @@ pg_stat_progress_cluster| SELECT s.pid,
s.param5 AS heap_tuples_written,
s.param6 AS heap_blks_total,
s.param7 AS heap_blks_scanned,
- s.param8 AS index_rebuild_count
+ s.param8 AS index_rebuild_count,
+ s.param18 AS partitions_total,
+ s.param19 AS partitions_done
FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
LEFT JOIN pg_database d ON ((s.datid = d.oid)));
pg_stat_progress_copy| SELECT s.pid,
--
2.17.0
--fmvA4kSBHQVZhkR6
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v9-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"
^ permalink raw reply [nested|flat] 73+ messages in thread
* [PATCH v9 3/8] f! progress reporting..
@ 2021-02-09 02:45 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 73+ messages in thread
From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)
This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.
This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
doc/src/sgml/monitoring.sgml | 21 +++++++++++
src/backend/catalog/system_views.sql | 4 +-
src/backend/commands/cluster.c | 55 ++++++++++++++++------------
src/backend/commands/vacuum.c | 5 +++
src/include/commands/progress.h | 3 ++
src/test/regress/expected/rules.out | 4 +-
6 files changed, 67 insertions(+), 25 deletions(-)
diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml
index 3513e127b7..dca47f6a48 100644
--- a/doc/src/sgml/monitoring.sgml
+++ b/doc/src/sgml/monitoring.sgml
@@ -6231,6 +6231,27 @@ SELECT pg_stat_get_backend_pid(s.backendid) AS pid,
is <literal>rebuilding index</literal>.
</para></entry>
</row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>total_partitions</structfield> <type>bigint</type>
+ </para>
+ When clustering a partitioned table, this column is set to the total
+ number of partitions to be clustered.
+ <para>
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>done_partitions</structfield> <type>bigint</type>
+ </para>
+ <para>
+ When clustering a partitioned table, this column is set to the number
+ of partitions which have been clustered.
+ </para></entry>
+ </row>
+
</tbody>
</tgroup>
</table>
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fc94a73a54..21c27f0437 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1070,7 +1070,9 @@ CREATE VIEW pg_stat_progress_cluster AS
S.param5 AS heap_tuples_written,
S.param6 AS heap_blks_total,
S.param7 AS heap_blks_scanned,
- S.param8 AS index_rebuild_count
+ S.param8 AS index_rebuild_count,
+ S.param18 AS partitions_total,
+ S.param19 AS partitions_done
FROM pg_stat_get_progress_info('CLUSTER') AS S
LEFT JOIN pg_database D ON S.datid = D.oid;
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 0c08ac56dc..7d8d1d8a69 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
static List *get_tables_to_cluster(MemoryContext cluster_context);
static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
/*---------------------------------------------------------------------------
@@ -188,16 +188,26 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
/* close relation, keep lock till commit */
table_close(rel, ShareUpdateExclusiveLock);
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
{
/* Do the job. */
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_CLUSTER);
cluster_rel(tableOid, indexOid, ¶ms);
+ pgstat_progress_end_command();
}
else
{
List *rvs;
MemoryContext cluster_context;
+ int64 progress_val[2];
+ const int progress_index[2] = {
+ PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+ };
+
/* Refuse to hold strong locks in a user transaction */
PreventInTransactionBlock(isTopLevel, "CLUSTER");
@@ -206,7 +216,15 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
ALLOCSET_DEFAULT_SIZES);
rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
- cluster_multiple_rels(rvs, params.options);
+
+ /* Report command and number of partitions */
+ progress_val[0] = PROGRESS_CLUSTER_COMMAND_CLUSTER;
+ /* XXX: rvs includes the parent, which is not a "partition" */
+ progress_val[1] = list_length(rvs);
+
+ pgstat_progress_update_multi_param(2, progress_index, progress_val);
+ cluster_multiple_rels(rvs, params.options, true);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -245,7 +263,11 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
* cluster_context.
*/
rvs = get_tables_to_cluster(cluster_context);
- cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_CLUSTER);
+ cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -282,14 +304,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* Check for user-requested abort. */
CHECK_FOR_INTERRUPTS();
- pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
- if (OidIsValid(indexOid))
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_CLUSTER);
- else
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
/*
* We grab exclusive access to the target rel and index for the duration
* of the transaction. (This is redundant for the single-transaction
@@ -300,10 +314,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* If the table has gone away, we can skip processing it */
if (!OldHeap)
- {
- pgstat_progress_end_command();
return;
- }
/*
* Since we may open a new transaction for each relation, we have to check
@@ -319,7 +330,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!pg_class_ownercheck(tableOid, GetUserId()))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -334,7 +344,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (RELATION_IS_OTHER_TEMP(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -346,7 +355,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -357,7 +365,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!get_index_isclustered(indexOid))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
}
@@ -416,7 +423,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!RelationIsPopulated(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -424,7 +430,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -440,8 +445,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
rebuild_relation(OldHeap, indexOid, verbose);
/* NB: rebuild_relation does table_close() on OldHeap */
-
- pgstat_progress_end_command();
}
/*
@@ -1632,9 +1635,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
/* Cluster each relation in a separate transaction */
static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
{
ListCell *lc;
+ off_t childnum = 0;
/* Commit to get out of starting transaction */
PopActiveSnapshot();
@@ -1657,6 +1661,11 @@ cluster_multiple_rels(List *rvs, int options)
cluster_rel(rvtc->tableOid, rvtc->indexOid,
&cluster_params);
+ if (ispartitioned)
+ pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+ ++childnum);
+
+
PopActiveSnapshot();
CommitTransactionCommand();
}
diff --git a/src/backend/commands/vacuum.c b/src/backend/commands/vacuum.c
index c064352e23..05d52858b1 100644
--- a/src/backend/commands/vacuum.c
+++ b/src/backend/commands/vacuum.c
@@ -37,6 +37,7 @@
#include "catalog/pg_namespace.h"
#include "commands/cluster.h"
#include "commands/defrem.h"
+#include "commands/progress.h"
#include "commands/vacuum.h"
#include "miscadmin.h"
#include "nodes/makefuncs.h"
@@ -1937,7 +1938,11 @@ vacuum_rel(Oid relid, RangeVar *relation, VacuumParams *params)
cluster_params.options |= CLUOPT_VERBOSE;
/* VACUUM FULL is now a variant of CLUSTER; see cluster.c */
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, relid);
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
cluster_rel(relid, InvalidOid, &cluster_params);
+ pgstat_progress_end_command();
}
else
table_relation_vacuum(onerel, params, vac_strategy);
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
#define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS 5
#define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED 6
#define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT 7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL 17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE 18
/* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
#define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP 1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b1c9b7bdfe..c888415235 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1943,7 +1943,9 @@ pg_stat_progress_cluster| SELECT s.pid,
s.param5 AS heap_tuples_written,
s.param6 AS heap_blks_total,
s.param7 AS heap_blks_scanned,
- s.param8 AS index_rebuild_count
+ s.param8 AS index_rebuild_count,
+ s.param18 AS partitions_total,
+ s.param19 AS partitions_done
FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
LEFT JOIN pg_database d ON ((s.datid = d.oid)));
pg_stat_progress_copy| SELECT s.pid,
--
2.17.0
--fmvA4kSBHQVZhkR6
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v9-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"
^ permalink raw reply [nested|flat] 73+ messages in thread
* [PATCH v8 3/8] f! progress reporting..
@ 2021-02-09 02:45 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 73+ messages in thread
From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)
This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.
This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
src/backend/catalog/system_views.sql | 4 +-
src/backend/commands/cluster.c | 64 +++++++++++++++++-----------
src/include/commands/progress.h | 3 ++
src/test/regress/expected/rules.out | 4 +-
4 files changed, 47 insertions(+), 28 deletions(-)
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fa58afd9d7..9a3cb5f935 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1069,7 +1069,9 @@ CREATE VIEW pg_stat_progress_cluster AS
S.param5 AS heap_tuples_written,
S.param6 AS heap_blks_total,
S.param7 AS heap_blks_scanned,
- S.param8 AS index_rebuild_count
+ S.param8 AS index_rebuild_count,
+ S.param18 AS partitions_total,
+ S.param19 AS partitions_done
FROM pg_stat_get_progress_info('CLUSTER') AS S
LEFT JOIN pg_database D ON S.datid = D.oid;
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 9b6673867c..cb4fc350c6 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
static List *get_tables_to_cluster(MemoryContext cluster_context);
static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
/*---------------------------------------------------------------------------
@@ -188,16 +188,28 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
/* close relation, keep lock till commit */
table_close(rel, ShareUpdateExclusiveLock);
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
{
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ OidIsValid(indexOid) ? PROGRESS_CLUSTER_COMMAND_CLUSTER :
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
+
/* Do the job. */
cluster_rel(tableOid, indexOid, ¶ms);
+ pgstat_progress_end_command();
}
else
{
List *rvs;
MemoryContext cluster_context;
+ int64 progress_val[2];
+ const int progress_index[] = {
+ PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+ };
+
/* Refuse to hold strong locks in a user transaction */
PreventInTransactionBlock(isTopLevel, "CLUSTER");
@@ -206,7 +218,20 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
ALLOCSET_DEFAULT_SIZES);
rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
- cluster_multiple_rels(rvs, params.options);
+
+ /* Report vacuum full or cluster and number of partitions */
+ progress_val[0] = OidIsValid(indexOid) ?
+ PROGRESS_CLUSTER_COMMAND_CLUSTER :
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL;
+ progress_val[1] = list_length(rvs);
+// This is currently showing the total number of *tables*, including the
+// parent, and intermediate partitions, so the column should be renamed, or
+// we should count the number of leaf partitions
+
+ pgstat_progress_update_multi_param(2, progress_index, progress_val);
+
+ cluster_multiple_rels(rvs, params.options, true);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -245,7 +270,7 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
* cluster_context.
*/
rvs = get_tables_to_cluster(cluster_context);
- cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+ cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -282,14 +307,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* Check for user-requested abort. */
CHECK_FOR_INTERRUPTS();
- pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
- if (OidIsValid(indexOid))
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_CLUSTER);
- else
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
/*
* We grab exclusive access to the target rel and index for the duration
* of the transaction. (This is redundant for the single-transaction
@@ -300,10 +317,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* If the table has gone away, we can skip processing it */
if (!OldHeap)
- {
- pgstat_progress_end_command();
return;
- }
/*
* Since we may open a new transaction for each relation, we have to check
@@ -319,7 +333,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!pg_class_ownercheck(tableOid, GetUserId()))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -334,7 +347,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (RELATION_IS_OTHER_TEMP(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -346,7 +358,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -357,7 +368,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!get_index_isclustered(indexOid))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
}
@@ -416,7 +426,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!RelationIsPopulated(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -424,7 +433,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -440,8 +448,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
rebuild_relation(OldHeap, indexOid, verbose);
/* NB: rebuild_relation does table_close() on OldHeap */
-
- pgstat_progress_end_command();
}
/*
@@ -1366,7 +1372,7 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap,
ReindexParams reindex_params = {0};
int i;
- /* Report that we are now swapping relation files */
+ /* Report that we are now swapping relation files XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_SWAP_REL_FILES);
@@ -1417,13 +1423,13 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap,
else if (newrelpersistence == RELPERSISTENCE_PERMANENT)
reindex_flags |= REINDEX_REL_FORCE_INDEXES_PERMANENT;
- /* Report that we are now reindexing relations */
+ /* Report that we are now reindexing relations XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_REBUILD_INDEX);
reindex_relation(OIDOldHeap, reindex_flags, &reindex_params);
- /* Report that we are now doing clean up */
+ /* Report that we are now doing clean up XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_FINAL_CLEANUP);
@@ -1633,9 +1639,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
/* Cluster each relation in a separate transaction */
static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
{
ListCell *lc;
+ off_t childnum = 0;
/* Commit to get out of starting transaction */
PopActiveSnapshot();
@@ -1658,6 +1665,11 @@ cluster_multiple_rels(List *rvs, int options)
cluster_rel(rvtc->tableOid, rvtc->indexOid,
&cluster_params);
+ if (ispartitioned)
+ pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+ childnum++);
+
+
PopActiveSnapshot();
CommitTransactionCommand();
}
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
#define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS 5
#define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED 6
#define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT 7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL 17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE 18
/* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
#define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP 1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b632d9f2ea..d1fea532bc 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1941,7 +1941,9 @@ pg_stat_progress_cluster| SELECT s.pid,
s.param5 AS heap_tuples_written,
s.param6 AS heap_blks_total,
s.param7 AS heap_blks_scanned,
- s.param8 AS index_rebuild_count
+ s.param8 AS index_rebuild_count,
+ s.param18 AS partitions_total,
+ s.param19 AS partitions_done
FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
LEFT JOIN pg_database d ON ((s.datid = d.oid)));
pg_stat_progress_copy| SELECT s.pid,
--
2.17.0
--O5XBE6gyVG5Rl6Rj
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v8-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"
^ permalink raw reply [nested|flat] 73+ messages in thread
* [PATCH v9 3/8] f! progress reporting..
@ 2021-02-09 02:45 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 73+ messages in thread
From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)
This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.
This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
doc/src/sgml/monitoring.sgml | 21 +++++++++++
src/backend/catalog/system_views.sql | 4 +-
src/backend/commands/cluster.c | 55 ++++++++++++++++------------
src/backend/commands/vacuum.c | 5 +++
src/include/commands/progress.h | 3 ++
src/test/regress/expected/rules.out | 4 +-
6 files changed, 67 insertions(+), 25 deletions(-)
diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml
index 3513e127b7..dca47f6a48 100644
--- a/doc/src/sgml/monitoring.sgml
+++ b/doc/src/sgml/monitoring.sgml
@@ -6231,6 +6231,27 @@ SELECT pg_stat_get_backend_pid(s.backendid) AS pid,
is <literal>rebuilding index</literal>.
</para></entry>
</row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>total_partitions</structfield> <type>bigint</type>
+ </para>
+ When clustering a partitioned table, this column is set to the total
+ number of partitions to be clustered.
+ <para>
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>done_partitions</structfield> <type>bigint</type>
+ </para>
+ <para>
+ When clustering a partitioned table, this column is set to the number
+ of partitions which have been clustered.
+ </para></entry>
+ </row>
+
</tbody>
</tgroup>
</table>
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fc94a73a54..21c27f0437 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1070,7 +1070,9 @@ CREATE VIEW pg_stat_progress_cluster AS
S.param5 AS heap_tuples_written,
S.param6 AS heap_blks_total,
S.param7 AS heap_blks_scanned,
- S.param8 AS index_rebuild_count
+ S.param8 AS index_rebuild_count,
+ S.param18 AS partitions_total,
+ S.param19 AS partitions_done
FROM pg_stat_get_progress_info('CLUSTER') AS S
LEFT JOIN pg_database D ON S.datid = D.oid;
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 0c08ac56dc..7d8d1d8a69 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
static List *get_tables_to_cluster(MemoryContext cluster_context);
static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
/*---------------------------------------------------------------------------
@@ -188,16 +188,26 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
/* close relation, keep lock till commit */
table_close(rel, ShareUpdateExclusiveLock);
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
{
/* Do the job. */
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_CLUSTER);
cluster_rel(tableOid, indexOid, ¶ms);
+ pgstat_progress_end_command();
}
else
{
List *rvs;
MemoryContext cluster_context;
+ int64 progress_val[2];
+ const int progress_index[2] = {
+ PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+ };
+
/* Refuse to hold strong locks in a user transaction */
PreventInTransactionBlock(isTopLevel, "CLUSTER");
@@ -206,7 +216,15 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
ALLOCSET_DEFAULT_SIZES);
rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
- cluster_multiple_rels(rvs, params.options);
+
+ /* Report command and number of partitions */
+ progress_val[0] = PROGRESS_CLUSTER_COMMAND_CLUSTER;
+ /* XXX: rvs includes the parent, which is not a "partition" */
+ progress_val[1] = list_length(rvs);
+
+ pgstat_progress_update_multi_param(2, progress_index, progress_val);
+ cluster_multiple_rels(rvs, params.options, true);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -245,7 +263,11 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
* cluster_context.
*/
rvs = get_tables_to_cluster(cluster_context);
- cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_CLUSTER);
+ cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -282,14 +304,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* Check for user-requested abort. */
CHECK_FOR_INTERRUPTS();
- pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
- if (OidIsValid(indexOid))
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_CLUSTER);
- else
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
/*
* We grab exclusive access to the target rel and index for the duration
* of the transaction. (This is redundant for the single-transaction
@@ -300,10 +314,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* If the table has gone away, we can skip processing it */
if (!OldHeap)
- {
- pgstat_progress_end_command();
return;
- }
/*
* Since we may open a new transaction for each relation, we have to check
@@ -319,7 +330,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!pg_class_ownercheck(tableOid, GetUserId()))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -334,7 +344,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (RELATION_IS_OTHER_TEMP(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -346,7 +355,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -357,7 +365,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!get_index_isclustered(indexOid))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
}
@@ -416,7 +423,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!RelationIsPopulated(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -424,7 +430,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -440,8 +445,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
rebuild_relation(OldHeap, indexOid, verbose);
/* NB: rebuild_relation does table_close() on OldHeap */
-
- pgstat_progress_end_command();
}
/*
@@ -1632,9 +1635,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
/* Cluster each relation in a separate transaction */
static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
{
ListCell *lc;
+ off_t childnum = 0;
/* Commit to get out of starting transaction */
PopActiveSnapshot();
@@ -1657,6 +1661,11 @@ cluster_multiple_rels(List *rvs, int options)
cluster_rel(rvtc->tableOid, rvtc->indexOid,
&cluster_params);
+ if (ispartitioned)
+ pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+ ++childnum);
+
+
PopActiveSnapshot();
CommitTransactionCommand();
}
diff --git a/src/backend/commands/vacuum.c b/src/backend/commands/vacuum.c
index c064352e23..05d52858b1 100644
--- a/src/backend/commands/vacuum.c
+++ b/src/backend/commands/vacuum.c
@@ -37,6 +37,7 @@
#include "catalog/pg_namespace.h"
#include "commands/cluster.h"
#include "commands/defrem.h"
+#include "commands/progress.h"
#include "commands/vacuum.h"
#include "miscadmin.h"
#include "nodes/makefuncs.h"
@@ -1937,7 +1938,11 @@ vacuum_rel(Oid relid, RangeVar *relation, VacuumParams *params)
cluster_params.options |= CLUOPT_VERBOSE;
/* VACUUM FULL is now a variant of CLUSTER; see cluster.c */
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, relid);
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
cluster_rel(relid, InvalidOid, &cluster_params);
+ pgstat_progress_end_command();
}
else
table_relation_vacuum(onerel, params, vac_strategy);
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
#define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS 5
#define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED 6
#define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT 7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL 17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE 18
/* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
#define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP 1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b1c9b7bdfe..c888415235 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1943,7 +1943,9 @@ pg_stat_progress_cluster| SELECT s.pid,
s.param5 AS heap_tuples_written,
s.param6 AS heap_blks_total,
s.param7 AS heap_blks_scanned,
- s.param8 AS index_rebuild_count
+ s.param8 AS index_rebuild_count,
+ s.param18 AS partitions_total,
+ s.param19 AS partitions_done
FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
LEFT JOIN pg_database d ON ((s.datid = d.oid)));
pg_stat_progress_copy| SELECT s.pid,
--
2.17.0
--fmvA4kSBHQVZhkR6
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v9-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"
^ permalink raw reply [nested|flat] 73+ messages in thread
* [PATCH v8 3/8] f! progress reporting..
@ 2021-02-09 02:45 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 73+ messages in thread
From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)
This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.
This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
src/backend/catalog/system_views.sql | 4 +-
src/backend/commands/cluster.c | 64 +++++++++++++++++-----------
src/include/commands/progress.h | 3 ++
src/test/regress/expected/rules.out | 4 +-
4 files changed, 47 insertions(+), 28 deletions(-)
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fa58afd9d7..9a3cb5f935 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1069,7 +1069,9 @@ CREATE VIEW pg_stat_progress_cluster AS
S.param5 AS heap_tuples_written,
S.param6 AS heap_blks_total,
S.param7 AS heap_blks_scanned,
- S.param8 AS index_rebuild_count
+ S.param8 AS index_rebuild_count,
+ S.param18 AS partitions_total,
+ S.param19 AS partitions_done
FROM pg_stat_get_progress_info('CLUSTER') AS S
LEFT JOIN pg_database D ON S.datid = D.oid;
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 9b6673867c..cb4fc350c6 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
static List *get_tables_to_cluster(MemoryContext cluster_context);
static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
/*---------------------------------------------------------------------------
@@ -188,16 +188,28 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
/* close relation, keep lock till commit */
table_close(rel, ShareUpdateExclusiveLock);
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
{
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ OidIsValid(indexOid) ? PROGRESS_CLUSTER_COMMAND_CLUSTER :
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
+
/* Do the job. */
cluster_rel(tableOid, indexOid, ¶ms);
+ pgstat_progress_end_command();
}
else
{
List *rvs;
MemoryContext cluster_context;
+ int64 progress_val[2];
+ const int progress_index[] = {
+ PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+ };
+
/* Refuse to hold strong locks in a user transaction */
PreventInTransactionBlock(isTopLevel, "CLUSTER");
@@ -206,7 +218,20 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
ALLOCSET_DEFAULT_SIZES);
rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
- cluster_multiple_rels(rvs, params.options);
+
+ /* Report vacuum full or cluster and number of partitions */
+ progress_val[0] = OidIsValid(indexOid) ?
+ PROGRESS_CLUSTER_COMMAND_CLUSTER :
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL;
+ progress_val[1] = list_length(rvs);
+// This is currently showing the total number of *tables*, including the
+// parent, and intermediate partitions, so the column should be renamed, or
+// we should count the number of leaf partitions
+
+ pgstat_progress_update_multi_param(2, progress_index, progress_val);
+
+ cluster_multiple_rels(rvs, params.options, true);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -245,7 +270,7 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
* cluster_context.
*/
rvs = get_tables_to_cluster(cluster_context);
- cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+ cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -282,14 +307,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* Check for user-requested abort. */
CHECK_FOR_INTERRUPTS();
- pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
- if (OidIsValid(indexOid))
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_CLUSTER);
- else
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
/*
* We grab exclusive access to the target rel and index for the duration
* of the transaction. (This is redundant for the single-transaction
@@ -300,10 +317,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* If the table has gone away, we can skip processing it */
if (!OldHeap)
- {
- pgstat_progress_end_command();
return;
- }
/*
* Since we may open a new transaction for each relation, we have to check
@@ -319,7 +333,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!pg_class_ownercheck(tableOid, GetUserId()))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -334,7 +347,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (RELATION_IS_OTHER_TEMP(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -346,7 +358,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -357,7 +368,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!get_index_isclustered(indexOid))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
}
@@ -416,7 +426,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!RelationIsPopulated(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -424,7 +433,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -440,8 +448,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
rebuild_relation(OldHeap, indexOid, verbose);
/* NB: rebuild_relation does table_close() on OldHeap */
-
- pgstat_progress_end_command();
}
/*
@@ -1366,7 +1372,7 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap,
ReindexParams reindex_params = {0};
int i;
- /* Report that we are now swapping relation files */
+ /* Report that we are now swapping relation files XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_SWAP_REL_FILES);
@@ -1417,13 +1423,13 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap,
else if (newrelpersistence == RELPERSISTENCE_PERMANENT)
reindex_flags |= REINDEX_REL_FORCE_INDEXES_PERMANENT;
- /* Report that we are now reindexing relations */
+ /* Report that we are now reindexing relations XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_REBUILD_INDEX);
reindex_relation(OIDOldHeap, reindex_flags, &reindex_params);
- /* Report that we are now doing clean up */
+ /* Report that we are now doing clean up XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_FINAL_CLEANUP);
@@ -1633,9 +1639,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
/* Cluster each relation in a separate transaction */
static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
{
ListCell *lc;
+ off_t childnum = 0;
/* Commit to get out of starting transaction */
PopActiveSnapshot();
@@ -1658,6 +1665,11 @@ cluster_multiple_rels(List *rvs, int options)
cluster_rel(rvtc->tableOid, rvtc->indexOid,
&cluster_params);
+ if (ispartitioned)
+ pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+ childnum++);
+
+
PopActiveSnapshot();
CommitTransactionCommand();
}
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
#define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS 5
#define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED 6
#define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT 7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL 17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE 18
/* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
#define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP 1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b632d9f2ea..d1fea532bc 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1941,7 +1941,9 @@ pg_stat_progress_cluster| SELECT s.pid,
s.param5 AS heap_tuples_written,
s.param6 AS heap_blks_total,
s.param7 AS heap_blks_scanned,
- s.param8 AS index_rebuild_count
+ s.param8 AS index_rebuild_count,
+ s.param18 AS partitions_total,
+ s.param19 AS partitions_done
FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
LEFT JOIN pg_database d ON ((s.datid = d.oid)));
pg_stat_progress_copy| SELECT s.pid,
--
2.17.0
--O5XBE6gyVG5Rl6Rj
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v8-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"
^ permalink raw reply [nested|flat] 73+ messages in thread
* [PATCH v8 3/8] f! progress reporting..
@ 2021-02-09 02:45 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 73+ messages in thread
From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)
This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.
This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
src/backend/catalog/system_views.sql | 4 +-
src/backend/commands/cluster.c | 64 +++++++++++++++++-----------
src/include/commands/progress.h | 3 ++
src/test/regress/expected/rules.out | 4 +-
4 files changed, 47 insertions(+), 28 deletions(-)
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fa58afd9d7..9a3cb5f935 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1069,7 +1069,9 @@ CREATE VIEW pg_stat_progress_cluster AS
S.param5 AS heap_tuples_written,
S.param6 AS heap_blks_total,
S.param7 AS heap_blks_scanned,
- S.param8 AS index_rebuild_count
+ S.param8 AS index_rebuild_count,
+ S.param18 AS partitions_total,
+ S.param19 AS partitions_done
FROM pg_stat_get_progress_info('CLUSTER') AS S
LEFT JOIN pg_database D ON S.datid = D.oid;
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 9b6673867c..cb4fc350c6 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
static List *get_tables_to_cluster(MemoryContext cluster_context);
static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
/*---------------------------------------------------------------------------
@@ -188,16 +188,28 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
/* close relation, keep lock till commit */
table_close(rel, ShareUpdateExclusiveLock);
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
{
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ OidIsValid(indexOid) ? PROGRESS_CLUSTER_COMMAND_CLUSTER :
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
+
/* Do the job. */
cluster_rel(tableOid, indexOid, ¶ms);
+ pgstat_progress_end_command();
}
else
{
List *rvs;
MemoryContext cluster_context;
+ int64 progress_val[2];
+ const int progress_index[] = {
+ PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+ };
+
/* Refuse to hold strong locks in a user transaction */
PreventInTransactionBlock(isTopLevel, "CLUSTER");
@@ -206,7 +218,20 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
ALLOCSET_DEFAULT_SIZES);
rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
- cluster_multiple_rels(rvs, params.options);
+
+ /* Report vacuum full or cluster and number of partitions */
+ progress_val[0] = OidIsValid(indexOid) ?
+ PROGRESS_CLUSTER_COMMAND_CLUSTER :
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL;
+ progress_val[1] = list_length(rvs);
+// This is currently showing the total number of *tables*, including the
+// parent, and intermediate partitions, so the column should be renamed, or
+// we should count the number of leaf partitions
+
+ pgstat_progress_update_multi_param(2, progress_index, progress_val);
+
+ cluster_multiple_rels(rvs, params.options, true);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -245,7 +270,7 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
* cluster_context.
*/
rvs = get_tables_to_cluster(cluster_context);
- cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+ cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -282,14 +307,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* Check for user-requested abort. */
CHECK_FOR_INTERRUPTS();
- pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
- if (OidIsValid(indexOid))
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_CLUSTER);
- else
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
/*
* We grab exclusive access to the target rel and index for the duration
* of the transaction. (This is redundant for the single-transaction
@@ -300,10 +317,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* If the table has gone away, we can skip processing it */
if (!OldHeap)
- {
- pgstat_progress_end_command();
return;
- }
/*
* Since we may open a new transaction for each relation, we have to check
@@ -319,7 +333,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!pg_class_ownercheck(tableOid, GetUserId()))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -334,7 +347,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (RELATION_IS_OTHER_TEMP(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -346,7 +358,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -357,7 +368,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!get_index_isclustered(indexOid))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
}
@@ -416,7 +426,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!RelationIsPopulated(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -424,7 +433,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -440,8 +448,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
rebuild_relation(OldHeap, indexOid, verbose);
/* NB: rebuild_relation does table_close() on OldHeap */
-
- pgstat_progress_end_command();
}
/*
@@ -1366,7 +1372,7 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap,
ReindexParams reindex_params = {0};
int i;
- /* Report that we are now swapping relation files */
+ /* Report that we are now swapping relation files XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_SWAP_REL_FILES);
@@ -1417,13 +1423,13 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap,
else if (newrelpersistence == RELPERSISTENCE_PERMANENT)
reindex_flags |= REINDEX_REL_FORCE_INDEXES_PERMANENT;
- /* Report that we are now reindexing relations */
+ /* Report that we are now reindexing relations XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_REBUILD_INDEX);
reindex_relation(OIDOldHeap, reindex_flags, &reindex_params);
- /* Report that we are now doing clean up */
+ /* Report that we are now doing clean up XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_FINAL_CLEANUP);
@@ -1633,9 +1639,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
/* Cluster each relation in a separate transaction */
static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
{
ListCell *lc;
+ off_t childnum = 0;
/* Commit to get out of starting transaction */
PopActiveSnapshot();
@@ -1658,6 +1665,11 @@ cluster_multiple_rels(List *rvs, int options)
cluster_rel(rvtc->tableOid, rvtc->indexOid,
&cluster_params);
+ if (ispartitioned)
+ pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+ childnum++);
+
+
PopActiveSnapshot();
CommitTransactionCommand();
}
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
#define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS 5
#define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED 6
#define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT 7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL 17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE 18
/* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
#define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP 1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b632d9f2ea..d1fea532bc 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1941,7 +1941,9 @@ pg_stat_progress_cluster| SELECT s.pid,
s.param5 AS heap_tuples_written,
s.param6 AS heap_blks_total,
s.param7 AS heap_blks_scanned,
- s.param8 AS index_rebuild_count
+ s.param8 AS index_rebuild_count,
+ s.param18 AS partitions_total,
+ s.param19 AS partitions_done
FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
LEFT JOIN pg_database d ON ((s.datid = d.oid)));
pg_stat_progress_copy| SELECT s.pid,
--
2.17.0
--O5XBE6gyVG5Rl6Rj
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v8-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"
^ permalink raw reply [nested|flat] 73+ messages in thread
* [PATCH v9 3/8] f! progress reporting..
@ 2021-02-09 02:45 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 73+ messages in thread
From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)
This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.
This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
doc/src/sgml/monitoring.sgml | 21 +++++++++++
src/backend/catalog/system_views.sql | 4 +-
src/backend/commands/cluster.c | 55 ++++++++++++++++------------
src/backend/commands/vacuum.c | 5 +++
src/include/commands/progress.h | 3 ++
src/test/regress/expected/rules.out | 4 +-
6 files changed, 67 insertions(+), 25 deletions(-)
diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml
index 3513e127b7..dca47f6a48 100644
--- a/doc/src/sgml/monitoring.sgml
+++ b/doc/src/sgml/monitoring.sgml
@@ -6231,6 +6231,27 @@ SELECT pg_stat_get_backend_pid(s.backendid) AS pid,
is <literal>rebuilding index</literal>.
</para></entry>
</row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>total_partitions</structfield> <type>bigint</type>
+ </para>
+ When clustering a partitioned table, this column is set to the total
+ number of partitions to be clustered.
+ <para>
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>done_partitions</structfield> <type>bigint</type>
+ </para>
+ <para>
+ When clustering a partitioned table, this column is set to the number
+ of partitions which have been clustered.
+ </para></entry>
+ </row>
+
</tbody>
</tgroup>
</table>
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fc94a73a54..21c27f0437 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1070,7 +1070,9 @@ CREATE VIEW pg_stat_progress_cluster AS
S.param5 AS heap_tuples_written,
S.param6 AS heap_blks_total,
S.param7 AS heap_blks_scanned,
- S.param8 AS index_rebuild_count
+ S.param8 AS index_rebuild_count,
+ S.param18 AS partitions_total,
+ S.param19 AS partitions_done
FROM pg_stat_get_progress_info('CLUSTER') AS S
LEFT JOIN pg_database D ON S.datid = D.oid;
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 0c08ac56dc..7d8d1d8a69 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
static List *get_tables_to_cluster(MemoryContext cluster_context);
static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
/*---------------------------------------------------------------------------
@@ -188,16 +188,26 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
/* close relation, keep lock till commit */
table_close(rel, ShareUpdateExclusiveLock);
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
{
/* Do the job. */
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_CLUSTER);
cluster_rel(tableOid, indexOid, ¶ms);
+ pgstat_progress_end_command();
}
else
{
List *rvs;
MemoryContext cluster_context;
+ int64 progress_val[2];
+ const int progress_index[2] = {
+ PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+ };
+
/* Refuse to hold strong locks in a user transaction */
PreventInTransactionBlock(isTopLevel, "CLUSTER");
@@ -206,7 +216,15 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
ALLOCSET_DEFAULT_SIZES);
rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
- cluster_multiple_rels(rvs, params.options);
+
+ /* Report command and number of partitions */
+ progress_val[0] = PROGRESS_CLUSTER_COMMAND_CLUSTER;
+ /* XXX: rvs includes the parent, which is not a "partition" */
+ progress_val[1] = list_length(rvs);
+
+ pgstat_progress_update_multi_param(2, progress_index, progress_val);
+ cluster_multiple_rels(rvs, params.options, true);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -245,7 +263,11 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
* cluster_context.
*/
rvs = get_tables_to_cluster(cluster_context);
- cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_CLUSTER);
+ cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -282,14 +304,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* Check for user-requested abort. */
CHECK_FOR_INTERRUPTS();
- pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
- if (OidIsValid(indexOid))
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_CLUSTER);
- else
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
/*
* We grab exclusive access to the target rel and index for the duration
* of the transaction. (This is redundant for the single-transaction
@@ -300,10 +314,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* If the table has gone away, we can skip processing it */
if (!OldHeap)
- {
- pgstat_progress_end_command();
return;
- }
/*
* Since we may open a new transaction for each relation, we have to check
@@ -319,7 +330,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!pg_class_ownercheck(tableOid, GetUserId()))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -334,7 +344,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (RELATION_IS_OTHER_TEMP(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -346,7 +355,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -357,7 +365,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!get_index_isclustered(indexOid))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
}
@@ -416,7 +423,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!RelationIsPopulated(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -424,7 +430,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -440,8 +445,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
rebuild_relation(OldHeap, indexOid, verbose);
/* NB: rebuild_relation does table_close() on OldHeap */
-
- pgstat_progress_end_command();
}
/*
@@ -1632,9 +1635,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
/* Cluster each relation in a separate transaction */
static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
{
ListCell *lc;
+ off_t childnum = 0;
/* Commit to get out of starting transaction */
PopActiveSnapshot();
@@ -1657,6 +1661,11 @@ cluster_multiple_rels(List *rvs, int options)
cluster_rel(rvtc->tableOid, rvtc->indexOid,
&cluster_params);
+ if (ispartitioned)
+ pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+ ++childnum);
+
+
PopActiveSnapshot();
CommitTransactionCommand();
}
diff --git a/src/backend/commands/vacuum.c b/src/backend/commands/vacuum.c
index c064352e23..05d52858b1 100644
--- a/src/backend/commands/vacuum.c
+++ b/src/backend/commands/vacuum.c
@@ -37,6 +37,7 @@
#include "catalog/pg_namespace.h"
#include "commands/cluster.h"
#include "commands/defrem.h"
+#include "commands/progress.h"
#include "commands/vacuum.h"
#include "miscadmin.h"
#include "nodes/makefuncs.h"
@@ -1937,7 +1938,11 @@ vacuum_rel(Oid relid, RangeVar *relation, VacuumParams *params)
cluster_params.options |= CLUOPT_VERBOSE;
/* VACUUM FULL is now a variant of CLUSTER; see cluster.c */
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, relid);
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
cluster_rel(relid, InvalidOid, &cluster_params);
+ pgstat_progress_end_command();
}
else
table_relation_vacuum(onerel, params, vac_strategy);
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
#define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS 5
#define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED 6
#define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT 7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL 17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE 18
/* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
#define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP 1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b1c9b7bdfe..c888415235 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1943,7 +1943,9 @@ pg_stat_progress_cluster| SELECT s.pid,
s.param5 AS heap_tuples_written,
s.param6 AS heap_blks_total,
s.param7 AS heap_blks_scanned,
- s.param8 AS index_rebuild_count
+ s.param8 AS index_rebuild_count,
+ s.param18 AS partitions_total,
+ s.param19 AS partitions_done
FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
LEFT JOIN pg_database d ON ((s.datid = d.oid)));
pg_stat_progress_copy| SELECT s.pid,
--
2.17.0
--fmvA4kSBHQVZhkR6
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v9-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"
^ permalink raw reply [nested|flat] 73+ messages in thread
* [PATCH v9 3/8] f! progress reporting..
@ 2021-02-09 02:45 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 73+ messages in thread
From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)
This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.
This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
doc/src/sgml/monitoring.sgml | 21 +++++++++++
src/backend/catalog/system_views.sql | 4 +-
src/backend/commands/cluster.c | 55 ++++++++++++++++------------
src/backend/commands/vacuum.c | 5 +++
src/include/commands/progress.h | 3 ++
src/test/regress/expected/rules.out | 4 +-
6 files changed, 67 insertions(+), 25 deletions(-)
diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml
index 3513e127b7..dca47f6a48 100644
--- a/doc/src/sgml/monitoring.sgml
+++ b/doc/src/sgml/monitoring.sgml
@@ -6231,6 +6231,27 @@ SELECT pg_stat_get_backend_pid(s.backendid) AS pid,
is <literal>rebuilding index</literal>.
</para></entry>
</row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>total_partitions</structfield> <type>bigint</type>
+ </para>
+ When clustering a partitioned table, this column is set to the total
+ number of partitions to be clustered.
+ <para>
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>done_partitions</structfield> <type>bigint</type>
+ </para>
+ <para>
+ When clustering a partitioned table, this column is set to the number
+ of partitions which have been clustered.
+ </para></entry>
+ </row>
+
</tbody>
</tgroup>
</table>
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fc94a73a54..21c27f0437 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1070,7 +1070,9 @@ CREATE VIEW pg_stat_progress_cluster AS
S.param5 AS heap_tuples_written,
S.param6 AS heap_blks_total,
S.param7 AS heap_blks_scanned,
- S.param8 AS index_rebuild_count
+ S.param8 AS index_rebuild_count,
+ S.param18 AS partitions_total,
+ S.param19 AS partitions_done
FROM pg_stat_get_progress_info('CLUSTER') AS S
LEFT JOIN pg_database D ON S.datid = D.oid;
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 0c08ac56dc..7d8d1d8a69 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
static List *get_tables_to_cluster(MemoryContext cluster_context);
static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
/*---------------------------------------------------------------------------
@@ -188,16 +188,26 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
/* close relation, keep lock till commit */
table_close(rel, ShareUpdateExclusiveLock);
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
{
/* Do the job. */
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_CLUSTER);
cluster_rel(tableOid, indexOid, ¶ms);
+ pgstat_progress_end_command();
}
else
{
List *rvs;
MemoryContext cluster_context;
+ int64 progress_val[2];
+ const int progress_index[2] = {
+ PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+ };
+
/* Refuse to hold strong locks in a user transaction */
PreventInTransactionBlock(isTopLevel, "CLUSTER");
@@ -206,7 +216,15 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
ALLOCSET_DEFAULT_SIZES);
rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
- cluster_multiple_rels(rvs, params.options);
+
+ /* Report command and number of partitions */
+ progress_val[0] = PROGRESS_CLUSTER_COMMAND_CLUSTER;
+ /* XXX: rvs includes the parent, which is not a "partition" */
+ progress_val[1] = list_length(rvs);
+
+ pgstat_progress_update_multi_param(2, progress_index, progress_val);
+ cluster_multiple_rels(rvs, params.options, true);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -245,7 +263,11 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
* cluster_context.
*/
rvs = get_tables_to_cluster(cluster_context);
- cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_CLUSTER);
+ cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -282,14 +304,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* Check for user-requested abort. */
CHECK_FOR_INTERRUPTS();
- pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
- if (OidIsValid(indexOid))
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_CLUSTER);
- else
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
/*
* We grab exclusive access to the target rel and index for the duration
* of the transaction. (This is redundant for the single-transaction
@@ -300,10 +314,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* If the table has gone away, we can skip processing it */
if (!OldHeap)
- {
- pgstat_progress_end_command();
return;
- }
/*
* Since we may open a new transaction for each relation, we have to check
@@ -319,7 +330,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!pg_class_ownercheck(tableOid, GetUserId()))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -334,7 +344,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (RELATION_IS_OTHER_TEMP(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -346,7 +355,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -357,7 +365,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!get_index_isclustered(indexOid))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
}
@@ -416,7 +423,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!RelationIsPopulated(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -424,7 +430,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -440,8 +445,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
rebuild_relation(OldHeap, indexOid, verbose);
/* NB: rebuild_relation does table_close() on OldHeap */
-
- pgstat_progress_end_command();
}
/*
@@ -1632,9 +1635,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
/* Cluster each relation in a separate transaction */
static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
{
ListCell *lc;
+ off_t childnum = 0;
/* Commit to get out of starting transaction */
PopActiveSnapshot();
@@ -1657,6 +1661,11 @@ cluster_multiple_rels(List *rvs, int options)
cluster_rel(rvtc->tableOid, rvtc->indexOid,
&cluster_params);
+ if (ispartitioned)
+ pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+ ++childnum);
+
+
PopActiveSnapshot();
CommitTransactionCommand();
}
diff --git a/src/backend/commands/vacuum.c b/src/backend/commands/vacuum.c
index c064352e23..05d52858b1 100644
--- a/src/backend/commands/vacuum.c
+++ b/src/backend/commands/vacuum.c
@@ -37,6 +37,7 @@
#include "catalog/pg_namespace.h"
#include "commands/cluster.h"
#include "commands/defrem.h"
+#include "commands/progress.h"
#include "commands/vacuum.h"
#include "miscadmin.h"
#include "nodes/makefuncs.h"
@@ -1937,7 +1938,11 @@ vacuum_rel(Oid relid, RangeVar *relation, VacuumParams *params)
cluster_params.options |= CLUOPT_VERBOSE;
/* VACUUM FULL is now a variant of CLUSTER; see cluster.c */
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, relid);
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
cluster_rel(relid, InvalidOid, &cluster_params);
+ pgstat_progress_end_command();
}
else
table_relation_vacuum(onerel, params, vac_strategy);
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
#define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS 5
#define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED 6
#define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT 7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL 17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE 18
/* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
#define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP 1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b1c9b7bdfe..c888415235 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1943,7 +1943,9 @@ pg_stat_progress_cluster| SELECT s.pid,
s.param5 AS heap_tuples_written,
s.param6 AS heap_blks_total,
s.param7 AS heap_blks_scanned,
- s.param8 AS index_rebuild_count
+ s.param8 AS index_rebuild_count,
+ s.param18 AS partitions_total,
+ s.param19 AS partitions_done
FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
LEFT JOIN pg_database d ON ((s.datid = d.oid)));
pg_stat_progress_copy| SELECT s.pid,
--
2.17.0
--fmvA4kSBHQVZhkR6
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v9-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"
^ permalink raw reply [nested|flat] 73+ messages in thread
* [PATCH v8 3/8] f! progress reporting..
@ 2021-02-09 02:45 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 73+ messages in thread
From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)
This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.
This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
src/backend/catalog/system_views.sql | 4 +-
src/backend/commands/cluster.c | 64 +++++++++++++++++-----------
src/include/commands/progress.h | 3 ++
src/test/regress/expected/rules.out | 4 +-
4 files changed, 47 insertions(+), 28 deletions(-)
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fa58afd9d7..9a3cb5f935 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1069,7 +1069,9 @@ CREATE VIEW pg_stat_progress_cluster AS
S.param5 AS heap_tuples_written,
S.param6 AS heap_blks_total,
S.param7 AS heap_blks_scanned,
- S.param8 AS index_rebuild_count
+ S.param8 AS index_rebuild_count,
+ S.param18 AS partitions_total,
+ S.param19 AS partitions_done
FROM pg_stat_get_progress_info('CLUSTER') AS S
LEFT JOIN pg_database D ON S.datid = D.oid;
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 9b6673867c..cb4fc350c6 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
static List *get_tables_to_cluster(MemoryContext cluster_context);
static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
/*---------------------------------------------------------------------------
@@ -188,16 +188,28 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
/* close relation, keep lock till commit */
table_close(rel, ShareUpdateExclusiveLock);
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
{
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ OidIsValid(indexOid) ? PROGRESS_CLUSTER_COMMAND_CLUSTER :
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
+
/* Do the job. */
cluster_rel(tableOid, indexOid, ¶ms);
+ pgstat_progress_end_command();
}
else
{
List *rvs;
MemoryContext cluster_context;
+ int64 progress_val[2];
+ const int progress_index[] = {
+ PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+ };
+
/* Refuse to hold strong locks in a user transaction */
PreventInTransactionBlock(isTopLevel, "CLUSTER");
@@ -206,7 +218,20 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
ALLOCSET_DEFAULT_SIZES);
rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
- cluster_multiple_rels(rvs, params.options);
+
+ /* Report vacuum full or cluster and number of partitions */
+ progress_val[0] = OidIsValid(indexOid) ?
+ PROGRESS_CLUSTER_COMMAND_CLUSTER :
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL;
+ progress_val[1] = list_length(rvs);
+// This is currently showing the total number of *tables*, including the
+// parent, and intermediate partitions, so the column should be renamed, or
+// we should count the number of leaf partitions
+
+ pgstat_progress_update_multi_param(2, progress_index, progress_val);
+
+ cluster_multiple_rels(rvs, params.options, true);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -245,7 +270,7 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
* cluster_context.
*/
rvs = get_tables_to_cluster(cluster_context);
- cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+ cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -282,14 +307,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* Check for user-requested abort. */
CHECK_FOR_INTERRUPTS();
- pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
- if (OidIsValid(indexOid))
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_CLUSTER);
- else
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
/*
* We grab exclusive access to the target rel and index for the duration
* of the transaction. (This is redundant for the single-transaction
@@ -300,10 +317,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* If the table has gone away, we can skip processing it */
if (!OldHeap)
- {
- pgstat_progress_end_command();
return;
- }
/*
* Since we may open a new transaction for each relation, we have to check
@@ -319,7 +333,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!pg_class_ownercheck(tableOid, GetUserId()))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -334,7 +347,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (RELATION_IS_OTHER_TEMP(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -346,7 +358,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -357,7 +368,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!get_index_isclustered(indexOid))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
}
@@ -416,7 +426,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!RelationIsPopulated(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -424,7 +433,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -440,8 +448,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
rebuild_relation(OldHeap, indexOid, verbose);
/* NB: rebuild_relation does table_close() on OldHeap */
-
- pgstat_progress_end_command();
}
/*
@@ -1366,7 +1372,7 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap,
ReindexParams reindex_params = {0};
int i;
- /* Report that we are now swapping relation files */
+ /* Report that we are now swapping relation files XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_SWAP_REL_FILES);
@@ -1417,13 +1423,13 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap,
else if (newrelpersistence == RELPERSISTENCE_PERMANENT)
reindex_flags |= REINDEX_REL_FORCE_INDEXES_PERMANENT;
- /* Report that we are now reindexing relations */
+ /* Report that we are now reindexing relations XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_REBUILD_INDEX);
reindex_relation(OIDOldHeap, reindex_flags, &reindex_params);
- /* Report that we are now doing clean up */
+ /* Report that we are now doing clean up XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_FINAL_CLEANUP);
@@ -1633,9 +1639,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
/* Cluster each relation in a separate transaction */
static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
{
ListCell *lc;
+ off_t childnum = 0;
/* Commit to get out of starting transaction */
PopActiveSnapshot();
@@ -1658,6 +1665,11 @@ cluster_multiple_rels(List *rvs, int options)
cluster_rel(rvtc->tableOid, rvtc->indexOid,
&cluster_params);
+ if (ispartitioned)
+ pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+ childnum++);
+
+
PopActiveSnapshot();
CommitTransactionCommand();
}
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
#define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS 5
#define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED 6
#define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT 7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL 17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE 18
/* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
#define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP 1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b632d9f2ea..d1fea532bc 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1941,7 +1941,9 @@ pg_stat_progress_cluster| SELECT s.pid,
s.param5 AS heap_tuples_written,
s.param6 AS heap_blks_total,
s.param7 AS heap_blks_scanned,
- s.param8 AS index_rebuild_count
+ s.param8 AS index_rebuild_count,
+ s.param18 AS partitions_total,
+ s.param19 AS partitions_done
FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
LEFT JOIN pg_database d ON ((s.datid = d.oid)));
pg_stat_progress_copy| SELECT s.pid,
--
2.17.0
--O5XBE6gyVG5Rl6Rj
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v8-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"
^ permalink raw reply [nested|flat] 73+ messages in thread
* [PATCH v8 3/8] f! progress reporting..
@ 2021-02-09 02:45 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 73+ messages in thread
From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)
This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.
This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
src/backend/catalog/system_views.sql | 4 +-
src/backend/commands/cluster.c | 64 +++++++++++++++++-----------
src/include/commands/progress.h | 3 ++
src/test/regress/expected/rules.out | 4 +-
4 files changed, 47 insertions(+), 28 deletions(-)
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fa58afd9d7..9a3cb5f935 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1069,7 +1069,9 @@ CREATE VIEW pg_stat_progress_cluster AS
S.param5 AS heap_tuples_written,
S.param6 AS heap_blks_total,
S.param7 AS heap_blks_scanned,
- S.param8 AS index_rebuild_count
+ S.param8 AS index_rebuild_count,
+ S.param18 AS partitions_total,
+ S.param19 AS partitions_done
FROM pg_stat_get_progress_info('CLUSTER') AS S
LEFT JOIN pg_database D ON S.datid = D.oid;
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 9b6673867c..cb4fc350c6 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
static List *get_tables_to_cluster(MemoryContext cluster_context);
static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
/*---------------------------------------------------------------------------
@@ -188,16 +188,28 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
/* close relation, keep lock till commit */
table_close(rel, ShareUpdateExclusiveLock);
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
{
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ OidIsValid(indexOid) ? PROGRESS_CLUSTER_COMMAND_CLUSTER :
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
+
/* Do the job. */
cluster_rel(tableOid, indexOid, ¶ms);
+ pgstat_progress_end_command();
}
else
{
List *rvs;
MemoryContext cluster_context;
+ int64 progress_val[2];
+ const int progress_index[] = {
+ PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+ };
+
/* Refuse to hold strong locks in a user transaction */
PreventInTransactionBlock(isTopLevel, "CLUSTER");
@@ -206,7 +218,20 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
ALLOCSET_DEFAULT_SIZES);
rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
- cluster_multiple_rels(rvs, params.options);
+
+ /* Report vacuum full or cluster and number of partitions */
+ progress_val[0] = OidIsValid(indexOid) ?
+ PROGRESS_CLUSTER_COMMAND_CLUSTER :
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL;
+ progress_val[1] = list_length(rvs);
+// This is currently showing the total number of *tables*, including the
+// parent, and intermediate partitions, so the column should be renamed, or
+// we should count the number of leaf partitions
+
+ pgstat_progress_update_multi_param(2, progress_index, progress_val);
+
+ cluster_multiple_rels(rvs, params.options, true);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -245,7 +270,7 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
* cluster_context.
*/
rvs = get_tables_to_cluster(cluster_context);
- cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+ cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -282,14 +307,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* Check for user-requested abort. */
CHECK_FOR_INTERRUPTS();
- pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
- if (OidIsValid(indexOid))
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_CLUSTER);
- else
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
/*
* We grab exclusive access to the target rel and index for the duration
* of the transaction. (This is redundant for the single-transaction
@@ -300,10 +317,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* If the table has gone away, we can skip processing it */
if (!OldHeap)
- {
- pgstat_progress_end_command();
return;
- }
/*
* Since we may open a new transaction for each relation, we have to check
@@ -319,7 +333,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!pg_class_ownercheck(tableOid, GetUserId()))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -334,7 +347,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (RELATION_IS_OTHER_TEMP(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -346,7 +358,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -357,7 +368,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!get_index_isclustered(indexOid))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
}
@@ -416,7 +426,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!RelationIsPopulated(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -424,7 +433,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -440,8 +448,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
rebuild_relation(OldHeap, indexOid, verbose);
/* NB: rebuild_relation does table_close() on OldHeap */
-
- pgstat_progress_end_command();
}
/*
@@ -1366,7 +1372,7 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap,
ReindexParams reindex_params = {0};
int i;
- /* Report that we are now swapping relation files */
+ /* Report that we are now swapping relation files XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_SWAP_REL_FILES);
@@ -1417,13 +1423,13 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap,
else if (newrelpersistence == RELPERSISTENCE_PERMANENT)
reindex_flags |= REINDEX_REL_FORCE_INDEXES_PERMANENT;
- /* Report that we are now reindexing relations */
+ /* Report that we are now reindexing relations XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_REBUILD_INDEX);
reindex_relation(OIDOldHeap, reindex_flags, &reindex_params);
- /* Report that we are now doing clean up */
+ /* Report that we are now doing clean up XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_FINAL_CLEANUP);
@@ -1633,9 +1639,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
/* Cluster each relation in a separate transaction */
static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
{
ListCell *lc;
+ off_t childnum = 0;
/* Commit to get out of starting transaction */
PopActiveSnapshot();
@@ -1658,6 +1665,11 @@ cluster_multiple_rels(List *rvs, int options)
cluster_rel(rvtc->tableOid, rvtc->indexOid,
&cluster_params);
+ if (ispartitioned)
+ pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+ childnum++);
+
+
PopActiveSnapshot();
CommitTransactionCommand();
}
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
#define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS 5
#define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED 6
#define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT 7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL 17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE 18
/* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
#define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP 1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b632d9f2ea..d1fea532bc 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1941,7 +1941,9 @@ pg_stat_progress_cluster| SELECT s.pid,
s.param5 AS heap_tuples_written,
s.param6 AS heap_blks_total,
s.param7 AS heap_blks_scanned,
- s.param8 AS index_rebuild_count
+ s.param8 AS index_rebuild_count,
+ s.param18 AS partitions_total,
+ s.param19 AS partitions_done
FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
LEFT JOIN pg_database d ON ((s.datid = d.oid)));
pg_stat_progress_copy| SELECT s.pid,
--
2.17.0
--O5XBE6gyVG5Rl6Rj
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v8-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"
^ permalink raw reply [nested|flat] 73+ messages in thread
* [PATCH v9 3/8] f! progress reporting..
@ 2021-02-09 02:45 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 73+ messages in thread
From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)
This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.
This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
doc/src/sgml/monitoring.sgml | 21 +++++++++++
src/backend/catalog/system_views.sql | 4 +-
src/backend/commands/cluster.c | 55 ++++++++++++++++------------
src/backend/commands/vacuum.c | 5 +++
src/include/commands/progress.h | 3 ++
src/test/regress/expected/rules.out | 4 +-
6 files changed, 67 insertions(+), 25 deletions(-)
diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml
index 3513e127b7..dca47f6a48 100644
--- a/doc/src/sgml/monitoring.sgml
+++ b/doc/src/sgml/monitoring.sgml
@@ -6231,6 +6231,27 @@ SELECT pg_stat_get_backend_pid(s.backendid) AS pid,
is <literal>rebuilding index</literal>.
</para></entry>
</row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>total_partitions</structfield> <type>bigint</type>
+ </para>
+ When clustering a partitioned table, this column is set to the total
+ number of partitions to be clustered.
+ <para>
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>done_partitions</structfield> <type>bigint</type>
+ </para>
+ <para>
+ When clustering a partitioned table, this column is set to the number
+ of partitions which have been clustered.
+ </para></entry>
+ </row>
+
</tbody>
</tgroup>
</table>
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fc94a73a54..21c27f0437 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1070,7 +1070,9 @@ CREATE VIEW pg_stat_progress_cluster AS
S.param5 AS heap_tuples_written,
S.param6 AS heap_blks_total,
S.param7 AS heap_blks_scanned,
- S.param8 AS index_rebuild_count
+ S.param8 AS index_rebuild_count,
+ S.param18 AS partitions_total,
+ S.param19 AS partitions_done
FROM pg_stat_get_progress_info('CLUSTER') AS S
LEFT JOIN pg_database D ON S.datid = D.oid;
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 0c08ac56dc..7d8d1d8a69 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
static List *get_tables_to_cluster(MemoryContext cluster_context);
static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
/*---------------------------------------------------------------------------
@@ -188,16 +188,26 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
/* close relation, keep lock till commit */
table_close(rel, ShareUpdateExclusiveLock);
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
{
/* Do the job. */
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_CLUSTER);
cluster_rel(tableOid, indexOid, ¶ms);
+ pgstat_progress_end_command();
}
else
{
List *rvs;
MemoryContext cluster_context;
+ int64 progress_val[2];
+ const int progress_index[2] = {
+ PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+ };
+
/* Refuse to hold strong locks in a user transaction */
PreventInTransactionBlock(isTopLevel, "CLUSTER");
@@ -206,7 +216,15 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
ALLOCSET_DEFAULT_SIZES);
rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
- cluster_multiple_rels(rvs, params.options);
+
+ /* Report command and number of partitions */
+ progress_val[0] = PROGRESS_CLUSTER_COMMAND_CLUSTER;
+ /* XXX: rvs includes the parent, which is not a "partition" */
+ progress_val[1] = list_length(rvs);
+
+ pgstat_progress_update_multi_param(2, progress_index, progress_val);
+ cluster_multiple_rels(rvs, params.options, true);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -245,7 +263,11 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
* cluster_context.
*/
rvs = get_tables_to_cluster(cluster_context);
- cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_CLUSTER);
+ cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -282,14 +304,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* Check for user-requested abort. */
CHECK_FOR_INTERRUPTS();
- pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
- if (OidIsValid(indexOid))
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_CLUSTER);
- else
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
/*
* We grab exclusive access to the target rel and index for the duration
* of the transaction. (This is redundant for the single-transaction
@@ -300,10 +314,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* If the table has gone away, we can skip processing it */
if (!OldHeap)
- {
- pgstat_progress_end_command();
return;
- }
/*
* Since we may open a new transaction for each relation, we have to check
@@ -319,7 +330,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!pg_class_ownercheck(tableOid, GetUserId()))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -334,7 +344,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (RELATION_IS_OTHER_TEMP(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -346,7 +355,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -357,7 +365,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!get_index_isclustered(indexOid))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
}
@@ -416,7 +423,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!RelationIsPopulated(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -424,7 +430,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -440,8 +445,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
rebuild_relation(OldHeap, indexOid, verbose);
/* NB: rebuild_relation does table_close() on OldHeap */
-
- pgstat_progress_end_command();
}
/*
@@ -1632,9 +1635,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
/* Cluster each relation in a separate transaction */
static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
{
ListCell *lc;
+ off_t childnum = 0;
/* Commit to get out of starting transaction */
PopActiveSnapshot();
@@ -1657,6 +1661,11 @@ cluster_multiple_rels(List *rvs, int options)
cluster_rel(rvtc->tableOid, rvtc->indexOid,
&cluster_params);
+ if (ispartitioned)
+ pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+ ++childnum);
+
+
PopActiveSnapshot();
CommitTransactionCommand();
}
diff --git a/src/backend/commands/vacuum.c b/src/backend/commands/vacuum.c
index c064352e23..05d52858b1 100644
--- a/src/backend/commands/vacuum.c
+++ b/src/backend/commands/vacuum.c
@@ -37,6 +37,7 @@
#include "catalog/pg_namespace.h"
#include "commands/cluster.h"
#include "commands/defrem.h"
+#include "commands/progress.h"
#include "commands/vacuum.h"
#include "miscadmin.h"
#include "nodes/makefuncs.h"
@@ -1937,7 +1938,11 @@ vacuum_rel(Oid relid, RangeVar *relation, VacuumParams *params)
cluster_params.options |= CLUOPT_VERBOSE;
/* VACUUM FULL is now a variant of CLUSTER; see cluster.c */
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, relid);
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
cluster_rel(relid, InvalidOid, &cluster_params);
+ pgstat_progress_end_command();
}
else
table_relation_vacuum(onerel, params, vac_strategy);
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
#define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS 5
#define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED 6
#define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT 7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL 17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE 18
/* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
#define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP 1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b1c9b7bdfe..c888415235 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1943,7 +1943,9 @@ pg_stat_progress_cluster| SELECT s.pid,
s.param5 AS heap_tuples_written,
s.param6 AS heap_blks_total,
s.param7 AS heap_blks_scanned,
- s.param8 AS index_rebuild_count
+ s.param8 AS index_rebuild_count,
+ s.param18 AS partitions_total,
+ s.param19 AS partitions_done
FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
LEFT JOIN pg_database d ON ((s.datid = d.oid)));
pg_stat_progress_copy| SELECT s.pid,
--
2.17.0
--fmvA4kSBHQVZhkR6
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v9-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"
^ permalink raw reply [nested|flat] 73+ messages in thread
* [PATCH v8 3/8] f! progress reporting..
@ 2021-02-09 02:45 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 73+ messages in thread
From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)
This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.
This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
src/backend/catalog/system_views.sql | 4 +-
src/backend/commands/cluster.c | 64 +++++++++++++++++-----------
src/include/commands/progress.h | 3 ++
src/test/regress/expected/rules.out | 4 +-
4 files changed, 47 insertions(+), 28 deletions(-)
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fa58afd9d7..9a3cb5f935 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1069,7 +1069,9 @@ CREATE VIEW pg_stat_progress_cluster AS
S.param5 AS heap_tuples_written,
S.param6 AS heap_blks_total,
S.param7 AS heap_blks_scanned,
- S.param8 AS index_rebuild_count
+ S.param8 AS index_rebuild_count,
+ S.param18 AS partitions_total,
+ S.param19 AS partitions_done
FROM pg_stat_get_progress_info('CLUSTER') AS S
LEFT JOIN pg_database D ON S.datid = D.oid;
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 9b6673867c..cb4fc350c6 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
static List *get_tables_to_cluster(MemoryContext cluster_context);
static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
/*---------------------------------------------------------------------------
@@ -188,16 +188,28 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
/* close relation, keep lock till commit */
table_close(rel, ShareUpdateExclusiveLock);
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
{
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ OidIsValid(indexOid) ? PROGRESS_CLUSTER_COMMAND_CLUSTER :
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
+
/* Do the job. */
cluster_rel(tableOid, indexOid, ¶ms);
+ pgstat_progress_end_command();
}
else
{
List *rvs;
MemoryContext cluster_context;
+ int64 progress_val[2];
+ const int progress_index[] = {
+ PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+ };
+
/* Refuse to hold strong locks in a user transaction */
PreventInTransactionBlock(isTopLevel, "CLUSTER");
@@ -206,7 +218,20 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
ALLOCSET_DEFAULT_SIZES);
rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
- cluster_multiple_rels(rvs, params.options);
+
+ /* Report vacuum full or cluster and number of partitions */
+ progress_val[0] = OidIsValid(indexOid) ?
+ PROGRESS_CLUSTER_COMMAND_CLUSTER :
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL;
+ progress_val[1] = list_length(rvs);
+// This is currently showing the total number of *tables*, including the
+// parent, and intermediate partitions, so the column should be renamed, or
+// we should count the number of leaf partitions
+
+ pgstat_progress_update_multi_param(2, progress_index, progress_val);
+
+ cluster_multiple_rels(rvs, params.options, true);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -245,7 +270,7 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
* cluster_context.
*/
rvs = get_tables_to_cluster(cluster_context);
- cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+ cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -282,14 +307,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* Check for user-requested abort. */
CHECK_FOR_INTERRUPTS();
- pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
- if (OidIsValid(indexOid))
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_CLUSTER);
- else
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
/*
* We grab exclusive access to the target rel and index for the duration
* of the transaction. (This is redundant for the single-transaction
@@ -300,10 +317,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* If the table has gone away, we can skip processing it */
if (!OldHeap)
- {
- pgstat_progress_end_command();
return;
- }
/*
* Since we may open a new transaction for each relation, we have to check
@@ -319,7 +333,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!pg_class_ownercheck(tableOid, GetUserId()))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -334,7 +347,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (RELATION_IS_OTHER_TEMP(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -346,7 +358,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -357,7 +368,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!get_index_isclustered(indexOid))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
}
@@ -416,7 +426,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!RelationIsPopulated(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -424,7 +433,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -440,8 +448,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
rebuild_relation(OldHeap, indexOid, verbose);
/* NB: rebuild_relation does table_close() on OldHeap */
-
- pgstat_progress_end_command();
}
/*
@@ -1366,7 +1372,7 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap,
ReindexParams reindex_params = {0};
int i;
- /* Report that we are now swapping relation files */
+ /* Report that we are now swapping relation files XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_SWAP_REL_FILES);
@@ -1417,13 +1423,13 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap,
else if (newrelpersistence == RELPERSISTENCE_PERMANENT)
reindex_flags |= REINDEX_REL_FORCE_INDEXES_PERMANENT;
- /* Report that we are now reindexing relations */
+ /* Report that we are now reindexing relations XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_REBUILD_INDEX);
reindex_relation(OIDOldHeap, reindex_flags, &reindex_params);
- /* Report that we are now doing clean up */
+ /* Report that we are now doing clean up XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_FINAL_CLEANUP);
@@ -1633,9 +1639,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
/* Cluster each relation in a separate transaction */
static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
{
ListCell *lc;
+ off_t childnum = 0;
/* Commit to get out of starting transaction */
PopActiveSnapshot();
@@ -1658,6 +1665,11 @@ cluster_multiple_rels(List *rvs, int options)
cluster_rel(rvtc->tableOid, rvtc->indexOid,
&cluster_params);
+ if (ispartitioned)
+ pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+ childnum++);
+
+
PopActiveSnapshot();
CommitTransactionCommand();
}
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
#define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS 5
#define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED 6
#define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT 7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL 17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE 18
/* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
#define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP 1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b632d9f2ea..d1fea532bc 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1941,7 +1941,9 @@ pg_stat_progress_cluster| SELECT s.pid,
s.param5 AS heap_tuples_written,
s.param6 AS heap_blks_total,
s.param7 AS heap_blks_scanned,
- s.param8 AS index_rebuild_count
+ s.param8 AS index_rebuild_count,
+ s.param18 AS partitions_total,
+ s.param19 AS partitions_done
FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
LEFT JOIN pg_database d ON ((s.datid = d.oid)));
pg_stat_progress_copy| SELECT s.pid,
--
2.17.0
--O5XBE6gyVG5Rl6Rj
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v8-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"
^ permalink raw reply [nested|flat] 73+ messages in thread
* [PATCH v9 3/8] f! progress reporting..
@ 2021-02-09 02:45 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 73+ messages in thread
From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)
This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.
This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
doc/src/sgml/monitoring.sgml | 21 +++++++++++
src/backend/catalog/system_views.sql | 4 +-
src/backend/commands/cluster.c | 55 ++++++++++++++++------------
src/backend/commands/vacuum.c | 5 +++
src/include/commands/progress.h | 3 ++
src/test/regress/expected/rules.out | 4 +-
6 files changed, 67 insertions(+), 25 deletions(-)
diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml
index 3513e127b7..dca47f6a48 100644
--- a/doc/src/sgml/monitoring.sgml
+++ b/doc/src/sgml/monitoring.sgml
@@ -6231,6 +6231,27 @@ SELECT pg_stat_get_backend_pid(s.backendid) AS pid,
is <literal>rebuilding index</literal>.
</para></entry>
</row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>total_partitions</structfield> <type>bigint</type>
+ </para>
+ When clustering a partitioned table, this column is set to the total
+ number of partitions to be clustered.
+ <para>
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>done_partitions</structfield> <type>bigint</type>
+ </para>
+ <para>
+ When clustering a partitioned table, this column is set to the number
+ of partitions which have been clustered.
+ </para></entry>
+ </row>
+
</tbody>
</tgroup>
</table>
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fc94a73a54..21c27f0437 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1070,7 +1070,9 @@ CREATE VIEW pg_stat_progress_cluster AS
S.param5 AS heap_tuples_written,
S.param6 AS heap_blks_total,
S.param7 AS heap_blks_scanned,
- S.param8 AS index_rebuild_count
+ S.param8 AS index_rebuild_count,
+ S.param18 AS partitions_total,
+ S.param19 AS partitions_done
FROM pg_stat_get_progress_info('CLUSTER') AS S
LEFT JOIN pg_database D ON S.datid = D.oid;
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 0c08ac56dc..7d8d1d8a69 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
static List *get_tables_to_cluster(MemoryContext cluster_context);
static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
/*---------------------------------------------------------------------------
@@ -188,16 +188,26 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
/* close relation, keep lock till commit */
table_close(rel, ShareUpdateExclusiveLock);
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
{
/* Do the job. */
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_CLUSTER);
cluster_rel(tableOid, indexOid, ¶ms);
+ pgstat_progress_end_command();
}
else
{
List *rvs;
MemoryContext cluster_context;
+ int64 progress_val[2];
+ const int progress_index[2] = {
+ PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+ };
+
/* Refuse to hold strong locks in a user transaction */
PreventInTransactionBlock(isTopLevel, "CLUSTER");
@@ -206,7 +216,15 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
ALLOCSET_DEFAULT_SIZES);
rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
- cluster_multiple_rels(rvs, params.options);
+
+ /* Report command and number of partitions */
+ progress_val[0] = PROGRESS_CLUSTER_COMMAND_CLUSTER;
+ /* XXX: rvs includes the parent, which is not a "partition" */
+ progress_val[1] = list_length(rvs);
+
+ pgstat_progress_update_multi_param(2, progress_index, progress_val);
+ cluster_multiple_rels(rvs, params.options, true);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -245,7 +263,11 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
* cluster_context.
*/
rvs = get_tables_to_cluster(cluster_context);
- cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_CLUSTER);
+ cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -282,14 +304,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* Check for user-requested abort. */
CHECK_FOR_INTERRUPTS();
- pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
- if (OidIsValid(indexOid))
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_CLUSTER);
- else
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
/*
* We grab exclusive access to the target rel and index for the duration
* of the transaction. (This is redundant for the single-transaction
@@ -300,10 +314,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* If the table has gone away, we can skip processing it */
if (!OldHeap)
- {
- pgstat_progress_end_command();
return;
- }
/*
* Since we may open a new transaction for each relation, we have to check
@@ -319,7 +330,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!pg_class_ownercheck(tableOid, GetUserId()))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -334,7 +344,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (RELATION_IS_OTHER_TEMP(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -346,7 +355,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -357,7 +365,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!get_index_isclustered(indexOid))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
}
@@ -416,7 +423,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!RelationIsPopulated(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -424,7 +430,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -440,8 +445,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
rebuild_relation(OldHeap, indexOid, verbose);
/* NB: rebuild_relation does table_close() on OldHeap */
-
- pgstat_progress_end_command();
}
/*
@@ -1632,9 +1635,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
/* Cluster each relation in a separate transaction */
static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
{
ListCell *lc;
+ off_t childnum = 0;
/* Commit to get out of starting transaction */
PopActiveSnapshot();
@@ -1657,6 +1661,11 @@ cluster_multiple_rels(List *rvs, int options)
cluster_rel(rvtc->tableOid, rvtc->indexOid,
&cluster_params);
+ if (ispartitioned)
+ pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+ ++childnum);
+
+
PopActiveSnapshot();
CommitTransactionCommand();
}
diff --git a/src/backend/commands/vacuum.c b/src/backend/commands/vacuum.c
index c064352e23..05d52858b1 100644
--- a/src/backend/commands/vacuum.c
+++ b/src/backend/commands/vacuum.c
@@ -37,6 +37,7 @@
#include "catalog/pg_namespace.h"
#include "commands/cluster.h"
#include "commands/defrem.h"
+#include "commands/progress.h"
#include "commands/vacuum.h"
#include "miscadmin.h"
#include "nodes/makefuncs.h"
@@ -1937,7 +1938,11 @@ vacuum_rel(Oid relid, RangeVar *relation, VacuumParams *params)
cluster_params.options |= CLUOPT_VERBOSE;
/* VACUUM FULL is now a variant of CLUSTER; see cluster.c */
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, relid);
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
cluster_rel(relid, InvalidOid, &cluster_params);
+ pgstat_progress_end_command();
}
else
table_relation_vacuum(onerel, params, vac_strategy);
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
#define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS 5
#define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED 6
#define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT 7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL 17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE 18
/* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
#define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP 1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b1c9b7bdfe..c888415235 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1943,7 +1943,9 @@ pg_stat_progress_cluster| SELECT s.pid,
s.param5 AS heap_tuples_written,
s.param6 AS heap_blks_total,
s.param7 AS heap_blks_scanned,
- s.param8 AS index_rebuild_count
+ s.param8 AS index_rebuild_count,
+ s.param18 AS partitions_total,
+ s.param19 AS partitions_done
FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
LEFT JOIN pg_database d ON ((s.datid = d.oid)));
pg_stat_progress_copy| SELECT s.pid,
--
2.17.0
--fmvA4kSBHQVZhkR6
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v9-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"
^ permalink raw reply [nested|flat] 73+ messages in thread
* [PATCH v8 3/8] f! progress reporting..
@ 2021-02-09 02:45 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 73+ messages in thread
From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)
This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.
This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
src/backend/catalog/system_views.sql | 4 +-
src/backend/commands/cluster.c | 64 +++++++++++++++++-----------
src/include/commands/progress.h | 3 ++
src/test/regress/expected/rules.out | 4 +-
4 files changed, 47 insertions(+), 28 deletions(-)
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fa58afd9d7..9a3cb5f935 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1069,7 +1069,9 @@ CREATE VIEW pg_stat_progress_cluster AS
S.param5 AS heap_tuples_written,
S.param6 AS heap_blks_total,
S.param7 AS heap_blks_scanned,
- S.param8 AS index_rebuild_count
+ S.param8 AS index_rebuild_count,
+ S.param18 AS partitions_total,
+ S.param19 AS partitions_done
FROM pg_stat_get_progress_info('CLUSTER') AS S
LEFT JOIN pg_database D ON S.datid = D.oid;
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 9b6673867c..cb4fc350c6 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
static List *get_tables_to_cluster(MemoryContext cluster_context);
static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
/*---------------------------------------------------------------------------
@@ -188,16 +188,28 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
/* close relation, keep lock till commit */
table_close(rel, ShareUpdateExclusiveLock);
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
{
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ OidIsValid(indexOid) ? PROGRESS_CLUSTER_COMMAND_CLUSTER :
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
+
/* Do the job. */
cluster_rel(tableOid, indexOid, ¶ms);
+ pgstat_progress_end_command();
}
else
{
List *rvs;
MemoryContext cluster_context;
+ int64 progress_val[2];
+ const int progress_index[] = {
+ PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+ };
+
/* Refuse to hold strong locks in a user transaction */
PreventInTransactionBlock(isTopLevel, "CLUSTER");
@@ -206,7 +218,20 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
ALLOCSET_DEFAULT_SIZES);
rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
- cluster_multiple_rels(rvs, params.options);
+
+ /* Report vacuum full or cluster and number of partitions */
+ progress_val[0] = OidIsValid(indexOid) ?
+ PROGRESS_CLUSTER_COMMAND_CLUSTER :
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL;
+ progress_val[1] = list_length(rvs);
+// This is currently showing the total number of *tables*, including the
+// parent, and intermediate partitions, so the column should be renamed, or
+// we should count the number of leaf partitions
+
+ pgstat_progress_update_multi_param(2, progress_index, progress_val);
+
+ cluster_multiple_rels(rvs, params.options, true);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -245,7 +270,7 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
* cluster_context.
*/
rvs = get_tables_to_cluster(cluster_context);
- cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+ cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -282,14 +307,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* Check for user-requested abort. */
CHECK_FOR_INTERRUPTS();
- pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
- if (OidIsValid(indexOid))
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_CLUSTER);
- else
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
/*
* We grab exclusive access to the target rel and index for the duration
* of the transaction. (This is redundant for the single-transaction
@@ -300,10 +317,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* If the table has gone away, we can skip processing it */
if (!OldHeap)
- {
- pgstat_progress_end_command();
return;
- }
/*
* Since we may open a new transaction for each relation, we have to check
@@ -319,7 +333,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!pg_class_ownercheck(tableOid, GetUserId()))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -334,7 +347,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (RELATION_IS_OTHER_TEMP(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -346,7 +358,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -357,7 +368,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!get_index_isclustered(indexOid))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
}
@@ -416,7 +426,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!RelationIsPopulated(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -424,7 +433,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -440,8 +448,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
rebuild_relation(OldHeap, indexOid, verbose);
/* NB: rebuild_relation does table_close() on OldHeap */
-
- pgstat_progress_end_command();
}
/*
@@ -1366,7 +1372,7 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap,
ReindexParams reindex_params = {0};
int i;
- /* Report that we are now swapping relation files */
+ /* Report that we are now swapping relation files XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_SWAP_REL_FILES);
@@ -1417,13 +1423,13 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap,
else if (newrelpersistence == RELPERSISTENCE_PERMANENT)
reindex_flags |= REINDEX_REL_FORCE_INDEXES_PERMANENT;
- /* Report that we are now reindexing relations */
+ /* Report that we are now reindexing relations XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_REBUILD_INDEX);
reindex_relation(OIDOldHeap, reindex_flags, &reindex_params);
- /* Report that we are now doing clean up */
+ /* Report that we are now doing clean up XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_FINAL_CLEANUP);
@@ -1633,9 +1639,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
/* Cluster each relation in a separate transaction */
static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
{
ListCell *lc;
+ off_t childnum = 0;
/* Commit to get out of starting transaction */
PopActiveSnapshot();
@@ -1658,6 +1665,11 @@ cluster_multiple_rels(List *rvs, int options)
cluster_rel(rvtc->tableOid, rvtc->indexOid,
&cluster_params);
+ if (ispartitioned)
+ pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+ childnum++);
+
+
PopActiveSnapshot();
CommitTransactionCommand();
}
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
#define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS 5
#define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED 6
#define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT 7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL 17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE 18
/* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
#define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP 1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b632d9f2ea..d1fea532bc 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1941,7 +1941,9 @@ pg_stat_progress_cluster| SELECT s.pid,
s.param5 AS heap_tuples_written,
s.param6 AS heap_blks_total,
s.param7 AS heap_blks_scanned,
- s.param8 AS index_rebuild_count
+ s.param8 AS index_rebuild_count,
+ s.param18 AS partitions_total,
+ s.param19 AS partitions_done
FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
LEFT JOIN pg_database d ON ((s.datid = d.oid)));
pg_stat_progress_copy| SELECT s.pid,
--
2.17.0
--O5XBE6gyVG5Rl6Rj
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v8-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"
^ permalink raw reply [nested|flat] 73+ messages in thread
* [PATCH v9 3/8] f! progress reporting..
@ 2021-02-09 02:45 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 73+ messages in thread
From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)
This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.
This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
doc/src/sgml/monitoring.sgml | 21 +++++++++++
src/backend/catalog/system_views.sql | 4 +-
src/backend/commands/cluster.c | 55 ++++++++++++++++------------
src/backend/commands/vacuum.c | 5 +++
src/include/commands/progress.h | 3 ++
src/test/regress/expected/rules.out | 4 +-
6 files changed, 67 insertions(+), 25 deletions(-)
diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml
index 3513e127b7..dca47f6a48 100644
--- a/doc/src/sgml/monitoring.sgml
+++ b/doc/src/sgml/monitoring.sgml
@@ -6231,6 +6231,27 @@ SELECT pg_stat_get_backend_pid(s.backendid) AS pid,
is <literal>rebuilding index</literal>.
</para></entry>
</row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>total_partitions</structfield> <type>bigint</type>
+ </para>
+ When clustering a partitioned table, this column is set to the total
+ number of partitions to be clustered.
+ <para>
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>done_partitions</structfield> <type>bigint</type>
+ </para>
+ <para>
+ When clustering a partitioned table, this column is set to the number
+ of partitions which have been clustered.
+ </para></entry>
+ </row>
+
</tbody>
</tgroup>
</table>
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fc94a73a54..21c27f0437 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1070,7 +1070,9 @@ CREATE VIEW pg_stat_progress_cluster AS
S.param5 AS heap_tuples_written,
S.param6 AS heap_blks_total,
S.param7 AS heap_blks_scanned,
- S.param8 AS index_rebuild_count
+ S.param8 AS index_rebuild_count,
+ S.param18 AS partitions_total,
+ S.param19 AS partitions_done
FROM pg_stat_get_progress_info('CLUSTER') AS S
LEFT JOIN pg_database D ON S.datid = D.oid;
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 0c08ac56dc..7d8d1d8a69 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
static List *get_tables_to_cluster(MemoryContext cluster_context);
static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
/*---------------------------------------------------------------------------
@@ -188,16 +188,26 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
/* close relation, keep lock till commit */
table_close(rel, ShareUpdateExclusiveLock);
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
{
/* Do the job. */
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_CLUSTER);
cluster_rel(tableOid, indexOid, ¶ms);
+ pgstat_progress_end_command();
}
else
{
List *rvs;
MemoryContext cluster_context;
+ int64 progress_val[2];
+ const int progress_index[2] = {
+ PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+ };
+
/* Refuse to hold strong locks in a user transaction */
PreventInTransactionBlock(isTopLevel, "CLUSTER");
@@ -206,7 +216,15 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
ALLOCSET_DEFAULT_SIZES);
rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
- cluster_multiple_rels(rvs, params.options);
+
+ /* Report command and number of partitions */
+ progress_val[0] = PROGRESS_CLUSTER_COMMAND_CLUSTER;
+ /* XXX: rvs includes the parent, which is not a "partition" */
+ progress_val[1] = list_length(rvs);
+
+ pgstat_progress_update_multi_param(2, progress_index, progress_val);
+ cluster_multiple_rels(rvs, params.options, true);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -245,7 +263,11 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
* cluster_context.
*/
rvs = get_tables_to_cluster(cluster_context);
- cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_CLUSTER);
+ cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -282,14 +304,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* Check for user-requested abort. */
CHECK_FOR_INTERRUPTS();
- pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
- if (OidIsValid(indexOid))
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_CLUSTER);
- else
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
/*
* We grab exclusive access to the target rel and index for the duration
* of the transaction. (This is redundant for the single-transaction
@@ -300,10 +314,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* If the table has gone away, we can skip processing it */
if (!OldHeap)
- {
- pgstat_progress_end_command();
return;
- }
/*
* Since we may open a new transaction for each relation, we have to check
@@ -319,7 +330,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!pg_class_ownercheck(tableOid, GetUserId()))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -334,7 +344,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (RELATION_IS_OTHER_TEMP(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -346,7 +355,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -357,7 +365,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!get_index_isclustered(indexOid))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
}
@@ -416,7 +423,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!RelationIsPopulated(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -424,7 +430,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -440,8 +445,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
rebuild_relation(OldHeap, indexOid, verbose);
/* NB: rebuild_relation does table_close() on OldHeap */
-
- pgstat_progress_end_command();
}
/*
@@ -1632,9 +1635,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
/* Cluster each relation in a separate transaction */
static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
{
ListCell *lc;
+ off_t childnum = 0;
/* Commit to get out of starting transaction */
PopActiveSnapshot();
@@ -1657,6 +1661,11 @@ cluster_multiple_rels(List *rvs, int options)
cluster_rel(rvtc->tableOid, rvtc->indexOid,
&cluster_params);
+ if (ispartitioned)
+ pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+ ++childnum);
+
+
PopActiveSnapshot();
CommitTransactionCommand();
}
diff --git a/src/backend/commands/vacuum.c b/src/backend/commands/vacuum.c
index c064352e23..05d52858b1 100644
--- a/src/backend/commands/vacuum.c
+++ b/src/backend/commands/vacuum.c
@@ -37,6 +37,7 @@
#include "catalog/pg_namespace.h"
#include "commands/cluster.h"
#include "commands/defrem.h"
+#include "commands/progress.h"
#include "commands/vacuum.h"
#include "miscadmin.h"
#include "nodes/makefuncs.h"
@@ -1937,7 +1938,11 @@ vacuum_rel(Oid relid, RangeVar *relation, VacuumParams *params)
cluster_params.options |= CLUOPT_VERBOSE;
/* VACUUM FULL is now a variant of CLUSTER; see cluster.c */
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, relid);
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
cluster_rel(relid, InvalidOid, &cluster_params);
+ pgstat_progress_end_command();
}
else
table_relation_vacuum(onerel, params, vac_strategy);
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
#define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS 5
#define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED 6
#define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT 7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL 17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE 18
/* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
#define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP 1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b1c9b7bdfe..c888415235 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1943,7 +1943,9 @@ pg_stat_progress_cluster| SELECT s.pid,
s.param5 AS heap_tuples_written,
s.param6 AS heap_blks_total,
s.param7 AS heap_blks_scanned,
- s.param8 AS index_rebuild_count
+ s.param8 AS index_rebuild_count,
+ s.param18 AS partitions_total,
+ s.param19 AS partitions_done
FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
LEFT JOIN pg_database d ON ((s.datid = d.oid)));
pg_stat_progress_copy| SELECT s.pid,
--
2.17.0
--fmvA4kSBHQVZhkR6
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v9-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"
^ permalink raw reply [nested|flat] 73+ messages in thread
* [PATCH v8 3/8] f! progress reporting..
@ 2021-02-09 02:45 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 73+ messages in thread
From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)
This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.
This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
src/backend/catalog/system_views.sql | 4 +-
src/backend/commands/cluster.c | 64 +++++++++++++++++-----------
src/include/commands/progress.h | 3 ++
src/test/regress/expected/rules.out | 4 +-
4 files changed, 47 insertions(+), 28 deletions(-)
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fa58afd9d7..9a3cb5f935 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1069,7 +1069,9 @@ CREATE VIEW pg_stat_progress_cluster AS
S.param5 AS heap_tuples_written,
S.param6 AS heap_blks_total,
S.param7 AS heap_blks_scanned,
- S.param8 AS index_rebuild_count
+ S.param8 AS index_rebuild_count,
+ S.param18 AS partitions_total,
+ S.param19 AS partitions_done
FROM pg_stat_get_progress_info('CLUSTER') AS S
LEFT JOIN pg_database D ON S.datid = D.oid;
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 9b6673867c..cb4fc350c6 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
static List *get_tables_to_cluster(MemoryContext cluster_context);
static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
/*---------------------------------------------------------------------------
@@ -188,16 +188,28 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
/* close relation, keep lock till commit */
table_close(rel, ShareUpdateExclusiveLock);
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
{
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ OidIsValid(indexOid) ? PROGRESS_CLUSTER_COMMAND_CLUSTER :
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
+
/* Do the job. */
cluster_rel(tableOid, indexOid, ¶ms);
+ pgstat_progress_end_command();
}
else
{
List *rvs;
MemoryContext cluster_context;
+ int64 progress_val[2];
+ const int progress_index[] = {
+ PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+ };
+
/* Refuse to hold strong locks in a user transaction */
PreventInTransactionBlock(isTopLevel, "CLUSTER");
@@ -206,7 +218,20 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
ALLOCSET_DEFAULT_SIZES);
rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
- cluster_multiple_rels(rvs, params.options);
+
+ /* Report vacuum full or cluster and number of partitions */
+ progress_val[0] = OidIsValid(indexOid) ?
+ PROGRESS_CLUSTER_COMMAND_CLUSTER :
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL;
+ progress_val[1] = list_length(rvs);
+// This is currently showing the total number of *tables*, including the
+// parent, and intermediate partitions, so the column should be renamed, or
+// we should count the number of leaf partitions
+
+ pgstat_progress_update_multi_param(2, progress_index, progress_val);
+
+ cluster_multiple_rels(rvs, params.options, true);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -245,7 +270,7 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
* cluster_context.
*/
rvs = get_tables_to_cluster(cluster_context);
- cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+ cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -282,14 +307,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* Check for user-requested abort. */
CHECK_FOR_INTERRUPTS();
- pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
- if (OidIsValid(indexOid))
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_CLUSTER);
- else
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
/*
* We grab exclusive access to the target rel and index for the duration
* of the transaction. (This is redundant for the single-transaction
@@ -300,10 +317,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* If the table has gone away, we can skip processing it */
if (!OldHeap)
- {
- pgstat_progress_end_command();
return;
- }
/*
* Since we may open a new transaction for each relation, we have to check
@@ -319,7 +333,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!pg_class_ownercheck(tableOid, GetUserId()))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -334,7 +347,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (RELATION_IS_OTHER_TEMP(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -346,7 +358,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -357,7 +368,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!get_index_isclustered(indexOid))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
}
@@ -416,7 +426,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!RelationIsPopulated(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -424,7 +433,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -440,8 +448,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
rebuild_relation(OldHeap, indexOid, verbose);
/* NB: rebuild_relation does table_close() on OldHeap */
-
- pgstat_progress_end_command();
}
/*
@@ -1366,7 +1372,7 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap,
ReindexParams reindex_params = {0};
int i;
- /* Report that we are now swapping relation files */
+ /* Report that we are now swapping relation files XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_SWAP_REL_FILES);
@@ -1417,13 +1423,13 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap,
else if (newrelpersistence == RELPERSISTENCE_PERMANENT)
reindex_flags |= REINDEX_REL_FORCE_INDEXES_PERMANENT;
- /* Report that we are now reindexing relations */
+ /* Report that we are now reindexing relations XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_REBUILD_INDEX);
reindex_relation(OIDOldHeap, reindex_flags, &reindex_params);
- /* Report that we are now doing clean up */
+ /* Report that we are now doing clean up XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_FINAL_CLEANUP);
@@ -1633,9 +1639,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
/* Cluster each relation in a separate transaction */
static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
{
ListCell *lc;
+ off_t childnum = 0;
/* Commit to get out of starting transaction */
PopActiveSnapshot();
@@ -1658,6 +1665,11 @@ cluster_multiple_rels(List *rvs, int options)
cluster_rel(rvtc->tableOid, rvtc->indexOid,
&cluster_params);
+ if (ispartitioned)
+ pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+ childnum++);
+
+
PopActiveSnapshot();
CommitTransactionCommand();
}
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
#define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS 5
#define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED 6
#define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT 7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL 17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE 18
/* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
#define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP 1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b632d9f2ea..d1fea532bc 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1941,7 +1941,9 @@ pg_stat_progress_cluster| SELECT s.pid,
s.param5 AS heap_tuples_written,
s.param6 AS heap_blks_total,
s.param7 AS heap_blks_scanned,
- s.param8 AS index_rebuild_count
+ s.param8 AS index_rebuild_count,
+ s.param18 AS partitions_total,
+ s.param19 AS partitions_done
FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
LEFT JOIN pg_database d ON ((s.datid = d.oid)));
pg_stat_progress_copy| SELECT s.pid,
--
2.17.0
--O5XBE6gyVG5Rl6Rj
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v8-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"
^ permalink raw reply [nested|flat] 73+ messages in thread
* [PATCH v9 3/8] f! progress reporting..
@ 2021-02-09 02:45 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 73+ messages in thread
From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)
This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.
This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
doc/src/sgml/monitoring.sgml | 21 +++++++++++
src/backend/catalog/system_views.sql | 4 +-
src/backend/commands/cluster.c | 55 ++++++++++++++++------------
src/backend/commands/vacuum.c | 5 +++
src/include/commands/progress.h | 3 ++
src/test/regress/expected/rules.out | 4 +-
6 files changed, 67 insertions(+), 25 deletions(-)
diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml
index 3513e127b7..dca47f6a48 100644
--- a/doc/src/sgml/monitoring.sgml
+++ b/doc/src/sgml/monitoring.sgml
@@ -6231,6 +6231,27 @@ SELECT pg_stat_get_backend_pid(s.backendid) AS pid,
is <literal>rebuilding index</literal>.
</para></entry>
</row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>total_partitions</structfield> <type>bigint</type>
+ </para>
+ When clustering a partitioned table, this column is set to the total
+ number of partitions to be clustered.
+ <para>
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>done_partitions</structfield> <type>bigint</type>
+ </para>
+ <para>
+ When clustering a partitioned table, this column is set to the number
+ of partitions which have been clustered.
+ </para></entry>
+ </row>
+
</tbody>
</tgroup>
</table>
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fc94a73a54..21c27f0437 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1070,7 +1070,9 @@ CREATE VIEW pg_stat_progress_cluster AS
S.param5 AS heap_tuples_written,
S.param6 AS heap_blks_total,
S.param7 AS heap_blks_scanned,
- S.param8 AS index_rebuild_count
+ S.param8 AS index_rebuild_count,
+ S.param18 AS partitions_total,
+ S.param19 AS partitions_done
FROM pg_stat_get_progress_info('CLUSTER') AS S
LEFT JOIN pg_database D ON S.datid = D.oid;
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 0c08ac56dc..7d8d1d8a69 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
static List *get_tables_to_cluster(MemoryContext cluster_context);
static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
/*---------------------------------------------------------------------------
@@ -188,16 +188,26 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
/* close relation, keep lock till commit */
table_close(rel, ShareUpdateExclusiveLock);
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
{
/* Do the job. */
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_CLUSTER);
cluster_rel(tableOid, indexOid, ¶ms);
+ pgstat_progress_end_command();
}
else
{
List *rvs;
MemoryContext cluster_context;
+ int64 progress_val[2];
+ const int progress_index[2] = {
+ PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+ };
+
/* Refuse to hold strong locks in a user transaction */
PreventInTransactionBlock(isTopLevel, "CLUSTER");
@@ -206,7 +216,15 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
ALLOCSET_DEFAULT_SIZES);
rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
- cluster_multiple_rels(rvs, params.options);
+
+ /* Report command and number of partitions */
+ progress_val[0] = PROGRESS_CLUSTER_COMMAND_CLUSTER;
+ /* XXX: rvs includes the parent, which is not a "partition" */
+ progress_val[1] = list_length(rvs);
+
+ pgstat_progress_update_multi_param(2, progress_index, progress_val);
+ cluster_multiple_rels(rvs, params.options, true);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -245,7 +263,11 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
* cluster_context.
*/
rvs = get_tables_to_cluster(cluster_context);
- cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_CLUSTER);
+ cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -282,14 +304,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* Check for user-requested abort. */
CHECK_FOR_INTERRUPTS();
- pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
- if (OidIsValid(indexOid))
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_CLUSTER);
- else
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
/*
* We grab exclusive access to the target rel and index for the duration
* of the transaction. (This is redundant for the single-transaction
@@ -300,10 +314,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* If the table has gone away, we can skip processing it */
if (!OldHeap)
- {
- pgstat_progress_end_command();
return;
- }
/*
* Since we may open a new transaction for each relation, we have to check
@@ -319,7 +330,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!pg_class_ownercheck(tableOid, GetUserId()))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -334,7 +344,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (RELATION_IS_OTHER_TEMP(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -346,7 +355,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -357,7 +365,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!get_index_isclustered(indexOid))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
}
@@ -416,7 +423,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!RelationIsPopulated(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -424,7 +430,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -440,8 +445,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
rebuild_relation(OldHeap, indexOid, verbose);
/* NB: rebuild_relation does table_close() on OldHeap */
-
- pgstat_progress_end_command();
}
/*
@@ -1632,9 +1635,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
/* Cluster each relation in a separate transaction */
static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
{
ListCell *lc;
+ off_t childnum = 0;
/* Commit to get out of starting transaction */
PopActiveSnapshot();
@@ -1657,6 +1661,11 @@ cluster_multiple_rels(List *rvs, int options)
cluster_rel(rvtc->tableOid, rvtc->indexOid,
&cluster_params);
+ if (ispartitioned)
+ pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+ ++childnum);
+
+
PopActiveSnapshot();
CommitTransactionCommand();
}
diff --git a/src/backend/commands/vacuum.c b/src/backend/commands/vacuum.c
index c064352e23..05d52858b1 100644
--- a/src/backend/commands/vacuum.c
+++ b/src/backend/commands/vacuum.c
@@ -37,6 +37,7 @@
#include "catalog/pg_namespace.h"
#include "commands/cluster.h"
#include "commands/defrem.h"
+#include "commands/progress.h"
#include "commands/vacuum.h"
#include "miscadmin.h"
#include "nodes/makefuncs.h"
@@ -1937,7 +1938,11 @@ vacuum_rel(Oid relid, RangeVar *relation, VacuumParams *params)
cluster_params.options |= CLUOPT_VERBOSE;
/* VACUUM FULL is now a variant of CLUSTER; see cluster.c */
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, relid);
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
cluster_rel(relid, InvalidOid, &cluster_params);
+ pgstat_progress_end_command();
}
else
table_relation_vacuum(onerel, params, vac_strategy);
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
#define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS 5
#define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED 6
#define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT 7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL 17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE 18
/* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
#define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP 1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b1c9b7bdfe..c888415235 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1943,7 +1943,9 @@ pg_stat_progress_cluster| SELECT s.pid,
s.param5 AS heap_tuples_written,
s.param6 AS heap_blks_total,
s.param7 AS heap_blks_scanned,
- s.param8 AS index_rebuild_count
+ s.param8 AS index_rebuild_count,
+ s.param18 AS partitions_total,
+ s.param19 AS partitions_done
FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
LEFT JOIN pg_database d ON ((s.datid = d.oid)));
pg_stat_progress_copy| SELECT s.pid,
--
2.17.0
--fmvA4kSBHQVZhkR6
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v9-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"
^ permalink raw reply [nested|flat] 73+ messages in thread
* [PATCH v8 3/8] f! progress reporting..
@ 2021-02-09 02:45 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 73+ messages in thread
From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)
This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.
This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
src/backend/catalog/system_views.sql | 4 +-
src/backend/commands/cluster.c | 64 +++++++++++++++++-----------
src/include/commands/progress.h | 3 ++
src/test/regress/expected/rules.out | 4 +-
4 files changed, 47 insertions(+), 28 deletions(-)
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fa58afd9d7..9a3cb5f935 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1069,7 +1069,9 @@ CREATE VIEW pg_stat_progress_cluster AS
S.param5 AS heap_tuples_written,
S.param6 AS heap_blks_total,
S.param7 AS heap_blks_scanned,
- S.param8 AS index_rebuild_count
+ S.param8 AS index_rebuild_count,
+ S.param18 AS partitions_total,
+ S.param19 AS partitions_done
FROM pg_stat_get_progress_info('CLUSTER') AS S
LEFT JOIN pg_database D ON S.datid = D.oid;
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 9b6673867c..cb4fc350c6 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
static List *get_tables_to_cluster(MemoryContext cluster_context);
static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
/*---------------------------------------------------------------------------
@@ -188,16 +188,28 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
/* close relation, keep lock till commit */
table_close(rel, ShareUpdateExclusiveLock);
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
{
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ OidIsValid(indexOid) ? PROGRESS_CLUSTER_COMMAND_CLUSTER :
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
+
/* Do the job. */
cluster_rel(tableOid, indexOid, ¶ms);
+ pgstat_progress_end_command();
}
else
{
List *rvs;
MemoryContext cluster_context;
+ int64 progress_val[2];
+ const int progress_index[] = {
+ PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+ };
+
/* Refuse to hold strong locks in a user transaction */
PreventInTransactionBlock(isTopLevel, "CLUSTER");
@@ -206,7 +218,20 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
ALLOCSET_DEFAULT_SIZES);
rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
- cluster_multiple_rels(rvs, params.options);
+
+ /* Report vacuum full or cluster and number of partitions */
+ progress_val[0] = OidIsValid(indexOid) ?
+ PROGRESS_CLUSTER_COMMAND_CLUSTER :
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL;
+ progress_val[1] = list_length(rvs);
+// This is currently showing the total number of *tables*, including the
+// parent, and intermediate partitions, so the column should be renamed, or
+// we should count the number of leaf partitions
+
+ pgstat_progress_update_multi_param(2, progress_index, progress_val);
+
+ cluster_multiple_rels(rvs, params.options, true);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -245,7 +270,7 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
* cluster_context.
*/
rvs = get_tables_to_cluster(cluster_context);
- cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+ cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -282,14 +307,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* Check for user-requested abort. */
CHECK_FOR_INTERRUPTS();
- pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
- if (OidIsValid(indexOid))
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_CLUSTER);
- else
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
/*
* We grab exclusive access to the target rel and index for the duration
* of the transaction. (This is redundant for the single-transaction
@@ -300,10 +317,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* If the table has gone away, we can skip processing it */
if (!OldHeap)
- {
- pgstat_progress_end_command();
return;
- }
/*
* Since we may open a new transaction for each relation, we have to check
@@ -319,7 +333,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!pg_class_ownercheck(tableOid, GetUserId()))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -334,7 +347,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (RELATION_IS_OTHER_TEMP(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -346,7 +358,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -357,7 +368,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!get_index_isclustered(indexOid))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
}
@@ -416,7 +426,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!RelationIsPopulated(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -424,7 +433,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -440,8 +448,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
rebuild_relation(OldHeap, indexOid, verbose);
/* NB: rebuild_relation does table_close() on OldHeap */
-
- pgstat_progress_end_command();
}
/*
@@ -1366,7 +1372,7 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap,
ReindexParams reindex_params = {0};
int i;
- /* Report that we are now swapping relation files */
+ /* Report that we are now swapping relation files XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_SWAP_REL_FILES);
@@ -1417,13 +1423,13 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap,
else if (newrelpersistence == RELPERSISTENCE_PERMANENT)
reindex_flags |= REINDEX_REL_FORCE_INDEXES_PERMANENT;
- /* Report that we are now reindexing relations */
+ /* Report that we are now reindexing relations XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_REBUILD_INDEX);
reindex_relation(OIDOldHeap, reindex_flags, &reindex_params);
- /* Report that we are now doing clean up */
+ /* Report that we are now doing clean up XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_FINAL_CLEANUP);
@@ -1633,9 +1639,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
/* Cluster each relation in a separate transaction */
static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
{
ListCell *lc;
+ off_t childnum = 0;
/* Commit to get out of starting transaction */
PopActiveSnapshot();
@@ -1658,6 +1665,11 @@ cluster_multiple_rels(List *rvs, int options)
cluster_rel(rvtc->tableOid, rvtc->indexOid,
&cluster_params);
+ if (ispartitioned)
+ pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+ childnum++);
+
+
PopActiveSnapshot();
CommitTransactionCommand();
}
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
#define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS 5
#define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED 6
#define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT 7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL 17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE 18
/* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
#define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP 1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b632d9f2ea..d1fea532bc 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1941,7 +1941,9 @@ pg_stat_progress_cluster| SELECT s.pid,
s.param5 AS heap_tuples_written,
s.param6 AS heap_blks_total,
s.param7 AS heap_blks_scanned,
- s.param8 AS index_rebuild_count
+ s.param8 AS index_rebuild_count,
+ s.param18 AS partitions_total,
+ s.param19 AS partitions_done
FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
LEFT JOIN pg_database d ON ((s.datid = d.oid)));
pg_stat_progress_copy| SELECT s.pid,
--
2.17.0
--O5XBE6gyVG5Rl6Rj
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v8-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"
^ permalink raw reply [nested|flat] 73+ messages in thread
* [PATCH v9 3/8] f! progress reporting..
@ 2021-02-09 02:45 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 73+ messages in thread
From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)
This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.
This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
doc/src/sgml/monitoring.sgml | 21 +++++++++++
src/backend/catalog/system_views.sql | 4 +-
src/backend/commands/cluster.c | 55 ++++++++++++++++------------
src/backend/commands/vacuum.c | 5 +++
src/include/commands/progress.h | 3 ++
src/test/regress/expected/rules.out | 4 +-
6 files changed, 67 insertions(+), 25 deletions(-)
diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml
index 3513e127b7..dca47f6a48 100644
--- a/doc/src/sgml/monitoring.sgml
+++ b/doc/src/sgml/monitoring.sgml
@@ -6231,6 +6231,27 @@ SELECT pg_stat_get_backend_pid(s.backendid) AS pid,
is <literal>rebuilding index</literal>.
</para></entry>
</row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>total_partitions</structfield> <type>bigint</type>
+ </para>
+ When clustering a partitioned table, this column is set to the total
+ number of partitions to be clustered.
+ <para>
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>done_partitions</structfield> <type>bigint</type>
+ </para>
+ <para>
+ When clustering a partitioned table, this column is set to the number
+ of partitions which have been clustered.
+ </para></entry>
+ </row>
+
</tbody>
</tgroup>
</table>
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fc94a73a54..21c27f0437 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1070,7 +1070,9 @@ CREATE VIEW pg_stat_progress_cluster AS
S.param5 AS heap_tuples_written,
S.param6 AS heap_blks_total,
S.param7 AS heap_blks_scanned,
- S.param8 AS index_rebuild_count
+ S.param8 AS index_rebuild_count,
+ S.param18 AS partitions_total,
+ S.param19 AS partitions_done
FROM pg_stat_get_progress_info('CLUSTER') AS S
LEFT JOIN pg_database D ON S.datid = D.oid;
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 0c08ac56dc..7d8d1d8a69 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
static List *get_tables_to_cluster(MemoryContext cluster_context);
static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
/*---------------------------------------------------------------------------
@@ -188,16 +188,26 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
/* close relation, keep lock till commit */
table_close(rel, ShareUpdateExclusiveLock);
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
{
/* Do the job. */
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_CLUSTER);
cluster_rel(tableOid, indexOid, ¶ms);
+ pgstat_progress_end_command();
}
else
{
List *rvs;
MemoryContext cluster_context;
+ int64 progress_val[2];
+ const int progress_index[2] = {
+ PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+ };
+
/* Refuse to hold strong locks in a user transaction */
PreventInTransactionBlock(isTopLevel, "CLUSTER");
@@ -206,7 +216,15 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
ALLOCSET_DEFAULT_SIZES);
rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
- cluster_multiple_rels(rvs, params.options);
+
+ /* Report command and number of partitions */
+ progress_val[0] = PROGRESS_CLUSTER_COMMAND_CLUSTER;
+ /* XXX: rvs includes the parent, which is not a "partition" */
+ progress_val[1] = list_length(rvs);
+
+ pgstat_progress_update_multi_param(2, progress_index, progress_val);
+ cluster_multiple_rels(rvs, params.options, true);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -245,7 +263,11 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
* cluster_context.
*/
rvs = get_tables_to_cluster(cluster_context);
- cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_CLUSTER);
+ cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -282,14 +304,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* Check for user-requested abort. */
CHECK_FOR_INTERRUPTS();
- pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
- if (OidIsValid(indexOid))
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_CLUSTER);
- else
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
/*
* We grab exclusive access to the target rel and index for the duration
* of the transaction. (This is redundant for the single-transaction
@@ -300,10 +314,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* If the table has gone away, we can skip processing it */
if (!OldHeap)
- {
- pgstat_progress_end_command();
return;
- }
/*
* Since we may open a new transaction for each relation, we have to check
@@ -319,7 +330,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!pg_class_ownercheck(tableOid, GetUserId()))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -334,7 +344,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (RELATION_IS_OTHER_TEMP(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -346,7 +355,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -357,7 +365,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!get_index_isclustered(indexOid))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
}
@@ -416,7 +423,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!RelationIsPopulated(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -424,7 +430,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -440,8 +445,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
rebuild_relation(OldHeap, indexOid, verbose);
/* NB: rebuild_relation does table_close() on OldHeap */
-
- pgstat_progress_end_command();
}
/*
@@ -1632,9 +1635,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
/* Cluster each relation in a separate transaction */
static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
{
ListCell *lc;
+ off_t childnum = 0;
/* Commit to get out of starting transaction */
PopActiveSnapshot();
@@ -1657,6 +1661,11 @@ cluster_multiple_rels(List *rvs, int options)
cluster_rel(rvtc->tableOid, rvtc->indexOid,
&cluster_params);
+ if (ispartitioned)
+ pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+ ++childnum);
+
+
PopActiveSnapshot();
CommitTransactionCommand();
}
diff --git a/src/backend/commands/vacuum.c b/src/backend/commands/vacuum.c
index c064352e23..05d52858b1 100644
--- a/src/backend/commands/vacuum.c
+++ b/src/backend/commands/vacuum.c
@@ -37,6 +37,7 @@
#include "catalog/pg_namespace.h"
#include "commands/cluster.h"
#include "commands/defrem.h"
+#include "commands/progress.h"
#include "commands/vacuum.h"
#include "miscadmin.h"
#include "nodes/makefuncs.h"
@@ -1937,7 +1938,11 @@ vacuum_rel(Oid relid, RangeVar *relation, VacuumParams *params)
cluster_params.options |= CLUOPT_VERBOSE;
/* VACUUM FULL is now a variant of CLUSTER; see cluster.c */
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, relid);
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
cluster_rel(relid, InvalidOid, &cluster_params);
+ pgstat_progress_end_command();
}
else
table_relation_vacuum(onerel, params, vac_strategy);
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
#define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS 5
#define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED 6
#define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT 7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL 17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE 18
/* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
#define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP 1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b1c9b7bdfe..c888415235 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1943,7 +1943,9 @@ pg_stat_progress_cluster| SELECT s.pid,
s.param5 AS heap_tuples_written,
s.param6 AS heap_blks_total,
s.param7 AS heap_blks_scanned,
- s.param8 AS index_rebuild_count
+ s.param8 AS index_rebuild_count,
+ s.param18 AS partitions_total,
+ s.param19 AS partitions_done
FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
LEFT JOIN pg_database d ON ((s.datid = d.oid)));
pg_stat_progress_copy| SELECT s.pid,
--
2.17.0
--fmvA4kSBHQVZhkR6
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v9-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"
^ permalink raw reply [nested|flat] 73+ messages in thread
* [PATCH v9 3/8] f! progress reporting..
@ 2021-02-09 02:45 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 73+ messages in thread
From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)
This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.
This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
doc/src/sgml/monitoring.sgml | 21 +++++++++++
src/backend/catalog/system_views.sql | 4 +-
src/backend/commands/cluster.c | 55 ++++++++++++++++------------
src/backend/commands/vacuum.c | 5 +++
src/include/commands/progress.h | 3 ++
src/test/regress/expected/rules.out | 4 +-
6 files changed, 67 insertions(+), 25 deletions(-)
diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml
index 3513e127b7..dca47f6a48 100644
--- a/doc/src/sgml/monitoring.sgml
+++ b/doc/src/sgml/monitoring.sgml
@@ -6231,6 +6231,27 @@ SELECT pg_stat_get_backend_pid(s.backendid) AS pid,
is <literal>rebuilding index</literal>.
</para></entry>
</row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>total_partitions</structfield> <type>bigint</type>
+ </para>
+ When clustering a partitioned table, this column is set to the total
+ number of partitions to be clustered.
+ <para>
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>done_partitions</structfield> <type>bigint</type>
+ </para>
+ <para>
+ When clustering a partitioned table, this column is set to the number
+ of partitions which have been clustered.
+ </para></entry>
+ </row>
+
</tbody>
</tgroup>
</table>
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fc94a73a54..21c27f0437 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1070,7 +1070,9 @@ CREATE VIEW pg_stat_progress_cluster AS
S.param5 AS heap_tuples_written,
S.param6 AS heap_blks_total,
S.param7 AS heap_blks_scanned,
- S.param8 AS index_rebuild_count
+ S.param8 AS index_rebuild_count,
+ S.param18 AS partitions_total,
+ S.param19 AS partitions_done
FROM pg_stat_get_progress_info('CLUSTER') AS S
LEFT JOIN pg_database D ON S.datid = D.oid;
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 0c08ac56dc..7d8d1d8a69 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
static List *get_tables_to_cluster(MemoryContext cluster_context);
static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
/*---------------------------------------------------------------------------
@@ -188,16 +188,26 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
/* close relation, keep lock till commit */
table_close(rel, ShareUpdateExclusiveLock);
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
{
/* Do the job. */
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_CLUSTER);
cluster_rel(tableOid, indexOid, ¶ms);
+ pgstat_progress_end_command();
}
else
{
List *rvs;
MemoryContext cluster_context;
+ int64 progress_val[2];
+ const int progress_index[2] = {
+ PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+ };
+
/* Refuse to hold strong locks in a user transaction */
PreventInTransactionBlock(isTopLevel, "CLUSTER");
@@ -206,7 +216,15 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
ALLOCSET_DEFAULT_SIZES);
rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
- cluster_multiple_rels(rvs, params.options);
+
+ /* Report command and number of partitions */
+ progress_val[0] = PROGRESS_CLUSTER_COMMAND_CLUSTER;
+ /* XXX: rvs includes the parent, which is not a "partition" */
+ progress_val[1] = list_length(rvs);
+
+ pgstat_progress_update_multi_param(2, progress_index, progress_val);
+ cluster_multiple_rels(rvs, params.options, true);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -245,7 +263,11 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
* cluster_context.
*/
rvs = get_tables_to_cluster(cluster_context);
- cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_CLUSTER);
+ cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -282,14 +304,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* Check for user-requested abort. */
CHECK_FOR_INTERRUPTS();
- pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
- if (OidIsValid(indexOid))
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_CLUSTER);
- else
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
/*
* We grab exclusive access to the target rel and index for the duration
* of the transaction. (This is redundant for the single-transaction
@@ -300,10 +314,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* If the table has gone away, we can skip processing it */
if (!OldHeap)
- {
- pgstat_progress_end_command();
return;
- }
/*
* Since we may open a new transaction for each relation, we have to check
@@ -319,7 +330,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!pg_class_ownercheck(tableOid, GetUserId()))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -334,7 +344,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (RELATION_IS_OTHER_TEMP(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -346,7 +355,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -357,7 +365,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!get_index_isclustered(indexOid))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
}
@@ -416,7 +423,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!RelationIsPopulated(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -424,7 +430,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -440,8 +445,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
rebuild_relation(OldHeap, indexOid, verbose);
/* NB: rebuild_relation does table_close() on OldHeap */
-
- pgstat_progress_end_command();
}
/*
@@ -1632,9 +1635,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
/* Cluster each relation in a separate transaction */
static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
{
ListCell *lc;
+ off_t childnum = 0;
/* Commit to get out of starting transaction */
PopActiveSnapshot();
@@ -1657,6 +1661,11 @@ cluster_multiple_rels(List *rvs, int options)
cluster_rel(rvtc->tableOid, rvtc->indexOid,
&cluster_params);
+ if (ispartitioned)
+ pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+ ++childnum);
+
+
PopActiveSnapshot();
CommitTransactionCommand();
}
diff --git a/src/backend/commands/vacuum.c b/src/backend/commands/vacuum.c
index c064352e23..05d52858b1 100644
--- a/src/backend/commands/vacuum.c
+++ b/src/backend/commands/vacuum.c
@@ -37,6 +37,7 @@
#include "catalog/pg_namespace.h"
#include "commands/cluster.h"
#include "commands/defrem.h"
+#include "commands/progress.h"
#include "commands/vacuum.h"
#include "miscadmin.h"
#include "nodes/makefuncs.h"
@@ -1937,7 +1938,11 @@ vacuum_rel(Oid relid, RangeVar *relation, VacuumParams *params)
cluster_params.options |= CLUOPT_VERBOSE;
/* VACUUM FULL is now a variant of CLUSTER; see cluster.c */
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, relid);
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
cluster_rel(relid, InvalidOid, &cluster_params);
+ pgstat_progress_end_command();
}
else
table_relation_vacuum(onerel, params, vac_strategy);
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
#define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS 5
#define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED 6
#define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT 7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL 17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE 18
/* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
#define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP 1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b1c9b7bdfe..c888415235 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1943,7 +1943,9 @@ pg_stat_progress_cluster| SELECT s.pid,
s.param5 AS heap_tuples_written,
s.param6 AS heap_blks_total,
s.param7 AS heap_blks_scanned,
- s.param8 AS index_rebuild_count
+ s.param8 AS index_rebuild_count,
+ s.param18 AS partitions_total,
+ s.param19 AS partitions_done
FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
LEFT JOIN pg_database d ON ((s.datid = d.oid)));
pg_stat_progress_copy| SELECT s.pid,
--
2.17.0
--fmvA4kSBHQVZhkR6
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v9-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"
^ permalink raw reply [nested|flat] 73+ messages in thread
* [PATCH v9 3/8] f! progress reporting..
@ 2021-02-09 02:45 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 73+ messages in thread
From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)
This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.
This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
doc/src/sgml/monitoring.sgml | 21 +++++++++++
src/backend/catalog/system_views.sql | 4 +-
src/backend/commands/cluster.c | 55 ++++++++++++++++------------
src/backend/commands/vacuum.c | 5 +++
src/include/commands/progress.h | 3 ++
src/test/regress/expected/rules.out | 4 +-
6 files changed, 67 insertions(+), 25 deletions(-)
diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml
index 3513e127b7..dca47f6a48 100644
--- a/doc/src/sgml/monitoring.sgml
+++ b/doc/src/sgml/monitoring.sgml
@@ -6231,6 +6231,27 @@ SELECT pg_stat_get_backend_pid(s.backendid) AS pid,
is <literal>rebuilding index</literal>.
</para></entry>
</row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>total_partitions</structfield> <type>bigint</type>
+ </para>
+ When clustering a partitioned table, this column is set to the total
+ number of partitions to be clustered.
+ <para>
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>done_partitions</structfield> <type>bigint</type>
+ </para>
+ <para>
+ When clustering a partitioned table, this column is set to the number
+ of partitions which have been clustered.
+ </para></entry>
+ </row>
+
</tbody>
</tgroup>
</table>
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fc94a73a54..21c27f0437 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1070,7 +1070,9 @@ CREATE VIEW pg_stat_progress_cluster AS
S.param5 AS heap_tuples_written,
S.param6 AS heap_blks_total,
S.param7 AS heap_blks_scanned,
- S.param8 AS index_rebuild_count
+ S.param8 AS index_rebuild_count,
+ S.param18 AS partitions_total,
+ S.param19 AS partitions_done
FROM pg_stat_get_progress_info('CLUSTER') AS S
LEFT JOIN pg_database D ON S.datid = D.oid;
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 0c08ac56dc..7d8d1d8a69 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
static List *get_tables_to_cluster(MemoryContext cluster_context);
static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
/*---------------------------------------------------------------------------
@@ -188,16 +188,26 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
/* close relation, keep lock till commit */
table_close(rel, ShareUpdateExclusiveLock);
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
{
/* Do the job. */
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_CLUSTER);
cluster_rel(tableOid, indexOid, ¶ms);
+ pgstat_progress_end_command();
}
else
{
List *rvs;
MemoryContext cluster_context;
+ int64 progress_val[2];
+ const int progress_index[2] = {
+ PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+ };
+
/* Refuse to hold strong locks in a user transaction */
PreventInTransactionBlock(isTopLevel, "CLUSTER");
@@ -206,7 +216,15 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
ALLOCSET_DEFAULT_SIZES);
rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
- cluster_multiple_rels(rvs, params.options);
+
+ /* Report command and number of partitions */
+ progress_val[0] = PROGRESS_CLUSTER_COMMAND_CLUSTER;
+ /* XXX: rvs includes the parent, which is not a "partition" */
+ progress_val[1] = list_length(rvs);
+
+ pgstat_progress_update_multi_param(2, progress_index, progress_val);
+ cluster_multiple_rels(rvs, params.options, true);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -245,7 +263,11 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
* cluster_context.
*/
rvs = get_tables_to_cluster(cluster_context);
- cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_CLUSTER);
+ cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -282,14 +304,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* Check for user-requested abort. */
CHECK_FOR_INTERRUPTS();
- pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
- if (OidIsValid(indexOid))
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_CLUSTER);
- else
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
/*
* We grab exclusive access to the target rel and index for the duration
* of the transaction. (This is redundant for the single-transaction
@@ -300,10 +314,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* If the table has gone away, we can skip processing it */
if (!OldHeap)
- {
- pgstat_progress_end_command();
return;
- }
/*
* Since we may open a new transaction for each relation, we have to check
@@ -319,7 +330,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!pg_class_ownercheck(tableOid, GetUserId()))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -334,7 +344,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (RELATION_IS_OTHER_TEMP(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -346,7 +355,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -357,7 +365,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!get_index_isclustered(indexOid))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
}
@@ -416,7 +423,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!RelationIsPopulated(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -424,7 +430,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -440,8 +445,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
rebuild_relation(OldHeap, indexOid, verbose);
/* NB: rebuild_relation does table_close() on OldHeap */
-
- pgstat_progress_end_command();
}
/*
@@ -1632,9 +1635,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
/* Cluster each relation in a separate transaction */
static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
{
ListCell *lc;
+ off_t childnum = 0;
/* Commit to get out of starting transaction */
PopActiveSnapshot();
@@ -1657,6 +1661,11 @@ cluster_multiple_rels(List *rvs, int options)
cluster_rel(rvtc->tableOid, rvtc->indexOid,
&cluster_params);
+ if (ispartitioned)
+ pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+ ++childnum);
+
+
PopActiveSnapshot();
CommitTransactionCommand();
}
diff --git a/src/backend/commands/vacuum.c b/src/backend/commands/vacuum.c
index c064352e23..05d52858b1 100644
--- a/src/backend/commands/vacuum.c
+++ b/src/backend/commands/vacuum.c
@@ -37,6 +37,7 @@
#include "catalog/pg_namespace.h"
#include "commands/cluster.h"
#include "commands/defrem.h"
+#include "commands/progress.h"
#include "commands/vacuum.h"
#include "miscadmin.h"
#include "nodes/makefuncs.h"
@@ -1937,7 +1938,11 @@ vacuum_rel(Oid relid, RangeVar *relation, VacuumParams *params)
cluster_params.options |= CLUOPT_VERBOSE;
/* VACUUM FULL is now a variant of CLUSTER; see cluster.c */
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, relid);
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
cluster_rel(relid, InvalidOid, &cluster_params);
+ pgstat_progress_end_command();
}
else
table_relation_vacuum(onerel, params, vac_strategy);
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
#define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS 5
#define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED 6
#define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT 7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL 17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE 18
/* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
#define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP 1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b1c9b7bdfe..c888415235 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1943,7 +1943,9 @@ pg_stat_progress_cluster| SELECT s.pid,
s.param5 AS heap_tuples_written,
s.param6 AS heap_blks_total,
s.param7 AS heap_blks_scanned,
- s.param8 AS index_rebuild_count
+ s.param8 AS index_rebuild_count,
+ s.param18 AS partitions_total,
+ s.param19 AS partitions_done
FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
LEFT JOIN pg_database d ON ((s.datid = d.oid)));
pg_stat_progress_copy| SELECT s.pid,
--
2.17.0
--fmvA4kSBHQVZhkR6
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v9-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"
^ permalink raw reply [nested|flat] 73+ messages in thread
* [PATCH v8 3/8] f! progress reporting..
@ 2021-02-09 02:45 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 73+ messages in thread
From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)
This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.
This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
src/backend/catalog/system_views.sql | 4 +-
src/backend/commands/cluster.c | 64 +++++++++++++++++-----------
src/include/commands/progress.h | 3 ++
src/test/regress/expected/rules.out | 4 +-
4 files changed, 47 insertions(+), 28 deletions(-)
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fa58afd9d7..9a3cb5f935 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1069,7 +1069,9 @@ CREATE VIEW pg_stat_progress_cluster AS
S.param5 AS heap_tuples_written,
S.param6 AS heap_blks_total,
S.param7 AS heap_blks_scanned,
- S.param8 AS index_rebuild_count
+ S.param8 AS index_rebuild_count,
+ S.param18 AS partitions_total,
+ S.param19 AS partitions_done
FROM pg_stat_get_progress_info('CLUSTER') AS S
LEFT JOIN pg_database D ON S.datid = D.oid;
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 9b6673867c..cb4fc350c6 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
static List *get_tables_to_cluster(MemoryContext cluster_context);
static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
/*---------------------------------------------------------------------------
@@ -188,16 +188,28 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
/* close relation, keep lock till commit */
table_close(rel, ShareUpdateExclusiveLock);
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
{
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ OidIsValid(indexOid) ? PROGRESS_CLUSTER_COMMAND_CLUSTER :
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
+
/* Do the job. */
cluster_rel(tableOid, indexOid, ¶ms);
+ pgstat_progress_end_command();
}
else
{
List *rvs;
MemoryContext cluster_context;
+ int64 progress_val[2];
+ const int progress_index[] = {
+ PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+ };
+
/* Refuse to hold strong locks in a user transaction */
PreventInTransactionBlock(isTopLevel, "CLUSTER");
@@ -206,7 +218,20 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
ALLOCSET_DEFAULT_SIZES);
rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
- cluster_multiple_rels(rvs, params.options);
+
+ /* Report vacuum full or cluster and number of partitions */
+ progress_val[0] = OidIsValid(indexOid) ?
+ PROGRESS_CLUSTER_COMMAND_CLUSTER :
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL;
+ progress_val[1] = list_length(rvs);
+// This is currently showing the total number of *tables*, including the
+// parent, and intermediate partitions, so the column should be renamed, or
+// we should count the number of leaf partitions
+
+ pgstat_progress_update_multi_param(2, progress_index, progress_val);
+
+ cluster_multiple_rels(rvs, params.options, true);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -245,7 +270,7 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
* cluster_context.
*/
rvs = get_tables_to_cluster(cluster_context);
- cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+ cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -282,14 +307,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* Check for user-requested abort. */
CHECK_FOR_INTERRUPTS();
- pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
- if (OidIsValid(indexOid))
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_CLUSTER);
- else
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
/*
* We grab exclusive access to the target rel and index for the duration
* of the transaction. (This is redundant for the single-transaction
@@ -300,10 +317,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* If the table has gone away, we can skip processing it */
if (!OldHeap)
- {
- pgstat_progress_end_command();
return;
- }
/*
* Since we may open a new transaction for each relation, we have to check
@@ -319,7 +333,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!pg_class_ownercheck(tableOid, GetUserId()))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -334,7 +347,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (RELATION_IS_OTHER_TEMP(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -346,7 +358,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -357,7 +368,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!get_index_isclustered(indexOid))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
}
@@ -416,7 +426,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!RelationIsPopulated(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -424,7 +433,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -440,8 +448,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
rebuild_relation(OldHeap, indexOid, verbose);
/* NB: rebuild_relation does table_close() on OldHeap */
-
- pgstat_progress_end_command();
}
/*
@@ -1366,7 +1372,7 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap,
ReindexParams reindex_params = {0};
int i;
- /* Report that we are now swapping relation files */
+ /* Report that we are now swapping relation files XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_SWAP_REL_FILES);
@@ -1417,13 +1423,13 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap,
else if (newrelpersistence == RELPERSISTENCE_PERMANENT)
reindex_flags |= REINDEX_REL_FORCE_INDEXES_PERMANENT;
- /* Report that we are now reindexing relations */
+ /* Report that we are now reindexing relations XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_REBUILD_INDEX);
reindex_relation(OIDOldHeap, reindex_flags, &reindex_params);
- /* Report that we are now doing clean up */
+ /* Report that we are now doing clean up XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_FINAL_CLEANUP);
@@ -1633,9 +1639,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
/* Cluster each relation in a separate transaction */
static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
{
ListCell *lc;
+ off_t childnum = 0;
/* Commit to get out of starting transaction */
PopActiveSnapshot();
@@ -1658,6 +1665,11 @@ cluster_multiple_rels(List *rvs, int options)
cluster_rel(rvtc->tableOid, rvtc->indexOid,
&cluster_params);
+ if (ispartitioned)
+ pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+ childnum++);
+
+
PopActiveSnapshot();
CommitTransactionCommand();
}
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
#define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS 5
#define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED 6
#define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT 7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL 17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE 18
/* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
#define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP 1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b632d9f2ea..d1fea532bc 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1941,7 +1941,9 @@ pg_stat_progress_cluster| SELECT s.pid,
s.param5 AS heap_tuples_written,
s.param6 AS heap_blks_total,
s.param7 AS heap_blks_scanned,
- s.param8 AS index_rebuild_count
+ s.param8 AS index_rebuild_count,
+ s.param18 AS partitions_total,
+ s.param19 AS partitions_done
FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
LEFT JOIN pg_database d ON ((s.datid = d.oid)));
pg_stat_progress_copy| SELECT s.pid,
--
2.17.0
--O5XBE6gyVG5Rl6Rj
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v8-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"
^ permalink raw reply [nested|flat] 73+ messages in thread
* [PATCH v9 3/8] f! progress reporting..
@ 2021-02-09 02:45 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 73+ messages in thread
From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)
This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.
This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
doc/src/sgml/monitoring.sgml | 21 +++++++++++
src/backend/catalog/system_views.sql | 4 +-
src/backend/commands/cluster.c | 55 ++++++++++++++++------------
src/backend/commands/vacuum.c | 5 +++
src/include/commands/progress.h | 3 ++
src/test/regress/expected/rules.out | 4 +-
6 files changed, 67 insertions(+), 25 deletions(-)
diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml
index 3513e127b7..dca47f6a48 100644
--- a/doc/src/sgml/monitoring.sgml
+++ b/doc/src/sgml/monitoring.sgml
@@ -6231,6 +6231,27 @@ SELECT pg_stat_get_backend_pid(s.backendid) AS pid,
is <literal>rebuilding index</literal>.
</para></entry>
</row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>total_partitions</structfield> <type>bigint</type>
+ </para>
+ When clustering a partitioned table, this column is set to the total
+ number of partitions to be clustered.
+ <para>
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>done_partitions</structfield> <type>bigint</type>
+ </para>
+ <para>
+ When clustering a partitioned table, this column is set to the number
+ of partitions which have been clustered.
+ </para></entry>
+ </row>
+
</tbody>
</tgroup>
</table>
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fc94a73a54..21c27f0437 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1070,7 +1070,9 @@ CREATE VIEW pg_stat_progress_cluster AS
S.param5 AS heap_tuples_written,
S.param6 AS heap_blks_total,
S.param7 AS heap_blks_scanned,
- S.param8 AS index_rebuild_count
+ S.param8 AS index_rebuild_count,
+ S.param18 AS partitions_total,
+ S.param19 AS partitions_done
FROM pg_stat_get_progress_info('CLUSTER') AS S
LEFT JOIN pg_database D ON S.datid = D.oid;
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 0c08ac56dc..7d8d1d8a69 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
static List *get_tables_to_cluster(MemoryContext cluster_context);
static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
/*---------------------------------------------------------------------------
@@ -188,16 +188,26 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
/* close relation, keep lock till commit */
table_close(rel, ShareUpdateExclusiveLock);
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
{
/* Do the job. */
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_CLUSTER);
cluster_rel(tableOid, indexOid, ¶ms);
+ pgstat_progress_end_command();
}
else
{
List *rvs;
MemoryContext cluster_context;
+ int64 progress_val[2];
+ const int progress_index[2] = {
+ PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+ };
+
/* Refuse to hold strong locks in a user transaction */
PreventInTransactionBlock(isTopLevel, "CLUSTER");
@@ -206,7 +216,15 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
ALLOCSET_DEFAULT_SIZES);
rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
- cluster_multiple_rels(rvs, params.options);
+
+ /* Report command and number of partitions */
+ progress_val[0] = PROGRESS_CLUSTER_COMMAND_CLUSTER;
+ /* XXX: rvs includes the parent, which is not a "partition" */
+ progress_val[1] = list_length(rvs);
+
+ pgstat_progress_update_multi_param(2, progress_index, progress_val);
+ cluster_multiple_rels(rvs, params.options, true);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -245,7 +263,11 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
* cluster_context.
*/
rvs = get_tables_to_cluster(cluster_context);
- cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_CLUSTER);
+ cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -282,14 +304,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* Check for user-requested abort. */
CHECK_FOR_INTERRUPTS();
- pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
- if (OidIsValid(indexOid))
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_CLUSTER);
- else
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
/*
* We grab exclusive access to the target rel and index for the duration
* of the transaction. (This is redundant for the single-transaction
@@ -300,10 +314,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* If the table has gone away, we can skip processing it */
if (!OldHeap)
- {
- pgstat_progress_end_command();
return;
- }
/*
* Since we may open a new transaction for each relation, we have to check
@@ -319,7 +330,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!pg_class_ownercheck(tableOid, GetUserId()))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -334,7 +344,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (RELATION_IS_OTHER_TEMP(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -346,7 +355,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -357,7 +365,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!get_index_isclustered(indexOid))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
}
@@ -416,7 +423,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!RelationIsPopulated(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -424,7 +430,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -440,8 +445,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
rebuild_relation(OldHeap, indexOid, verbose);
/* NB: rebuild_relation does table_close() on OldHeap */
-
- pgstat_progress_end_command();
}
/*
@@ -1632,9 +1635,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
/* Cluster each relation in a separate transaction */
static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
{
ListCell *lc;
+ off_t childnum = 0;
/* Commit to get out of starting transaction */
PopActiveSnapshot();
@@ -1657,6 +1661,11 @@ cluster_multiple_rels(List *rvs, int options)
cluster_rel(rvtc->tableOid, rvtc->indexOid,
&cluster_params);
+ if (ispartitioned)
+ pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+ ++childnum);
+
+
PopActiveSnapshot();
CommitTransactionCommand();
}
diff --git a/src/backend/commands/vacuum.c b/src/backend/commands/vacuum.c
index c064352e23..05d52858b1 100644
--- a/src/backend/commands/vacuum.c
+++ b/src/backend/commands/vacuum.c
@@ -37,6 +37,7 @@
#include "catalog/pg_namespace.h"
#include "commands/cluster.h"
#include "commands/defrem.h"
+#include "commands/progress.h"
#include "commands/vacuum.h"
#include "miscadmin.h"
#include "nodes/makefuncs.h"
@@ -1937,7 +1938,11 @@ vacuum_rel(Oid relid, RangeVar *relation, VacuumParams *params)
cluster_params.options |= CLUOPT_VERBOSE;
/* VACUUM FULL is now a variant of CLUSTER; see cluster.c */
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, relid);
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
cluster_rel(relid, InvalidOid, &cluster_params);
+ pgstat_progress_end_command();
}
else
table_relation_vacuum(onerel, params, vac_strategy);
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
#define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS 5
#define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED 6
#define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT 7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL 17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE 18
/* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
#define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP 1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b1c9b7bdfe..c888415235 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1943,7 +1943,9 @@ pg_stat_progress_cluster| SELECT s.pid,
s.param5 AS heap_tuples_written,
s.param6 AS heap_blks_total,
s.param7 AS heap_blks_scanned,
- s.param8 AS index_rebuild_count
+ s.param8 AS index_rebuild_count,
+ s.param18 AS partitions_total,
+ s.param19 AS partitions_done
FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
LEFT JOIN pg_database d ON ((s.datid = d.oid)));
pg_stat_progress_copy| SELECT s.pid,
--
2.17.0
--fmvA4kSBHQVZhkR6
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v9-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"
^ permalink raw reply [nested|flat] 73+ messages in thread
* [PATCH v8 3/8] f! progress reporting..
@ 2021-02-09 02:45 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 73+ messages in thread
From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)
This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.
This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
src/backend/catalog/system_views.sql | 4 +-
src/backend/commands/cluster.c | 64 +++++++++++++++++-----------
src/include/commands/progress.h | 3 ++
src/test/regress/expected/rules.out | 4 +-
4 files changed, 47 insertions(+), 28 deletions(-)
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fa58afd9d7..9a3cb5f935 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1069,7 +1069,9 @@ CREATE VIEW pg_stat_progress_cluster AS
S.param5 AS heap_tuples_written,
S.param6 AS heap_blks_total,
S.param7 AS heap_blks_scanned,
- S.param8 AS index_rebuild_count
+ S.param8 AS index_rebuild_count,
+ S.param18 AS partitions_total,
+ S.param19 AS partitions_done
FROM pg_stat_get_progress_info('CLUSTER') AS S
LEFT JOIN pg_database D ON S.datid = D.oid;
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 9b6673867c..cb4fc350c6 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
static List *get_tables_to_cluster(MemoryContext cluster_context);
static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
/*---------------------------------------------------------------------------
@@ -188,16 +188,28 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
/* close relation, keep lock till commit */
table_close(rel, ShareUpdateExclusiveLock);
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
{
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ OidIsValid(indexOid) ? PROGRESS_CLUSTER_COMMAND_CLUSTER :
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
+
/* Do the job. */
cluster_rel(tableOid, indexOid, ¶ms);
+ pgstat_progress_end_command();
}
else
{
List *rvs;
MemoryContext cluster_context;
+ int64 progress_val[2];
+ const int progress_index[] = {
+ PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+ };
+
/* Refuse to hold strong locks in a user transaction */
PreventInTransactionBlock(isTopLevel, "CLUSTER");
@@ -206,7 +218,20 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
ALLOCSET_DEFAULT_SIZES);
rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
- cluster_multiple_rels(rvs, params.options);
+
+ /* Report vacuum full or cluster and number of partitions */
+ progress_val[0] = OidIsValid(indexOid) ?
+ PROGRESS_CLUSTER_COMMAND_CLUSTER :
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL;
+ progress_val[1] = list_length(rvs);
+// This is currently showing the total number of *tables*, including the
+// parent, and intermediate partitions, so the column should be renamed, or
+// we should count the number of leaf partitions
+
+ pgstat_progress_update_multi_param(2, progress_index, progress_val);
+
+ cluster_multiple_rels(rvs, params.options, true);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -245,7 +270,7 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
* cluster_context.
*/
rvs = get_tables_to_cluster(cluster_context);
- cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+ cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -282,14 +307,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* Check for user-requested abort. */
CHECK_FOR_INTERRUPTS();
- pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
- if (OidIsValid(indexOid))
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_CLUSTER);
- else
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
/*
* We grab exclusive access to the target rel and index for the duration
* of the transaction. (This is redundant for the single-transaction
@@ -300,10 +317,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* If the table has gone away, we can skip processing it */
if (!OldHeap)
- {
- pgstat_progress_end_command();
return;
- }
/*
* Since we may open a new transaction for each relation, we have to check
@@ -319,7 +333,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!pg_class_ownercheck(tableOid, GetUserId()))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -334,7 +347,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (RELATION_IS_OTHER_TEMP(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -346,7 +358,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -357,7 +368,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!get_index_isclustered(indexOid))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
}
@@ -416,7 +426,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!RelationIsPopulated(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -424,7 +433,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -440,8 +448,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
rebuild_relation(OldHeap, indexOid, verbose);
/* NB: rebuild_relation does table_close() on OldHeap */
-
- pgstat_progress_end_command();
}
/*
@@ -1366,7 +1372,7 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap,
ReindexParams reindex_params = {0};
int i;
- /* Report that we are now swapping relation files */
+ /* Report that we are now swapping relation files XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_SWAP_REL_FILES);
@@ -1417,13 +1423,13 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap,
else if (newrelpersistence == RELPERSISTENCE_PERMANENT)
reindex_flags |= REINDEX_REL_FORCE_INDEXES_PERMANENT;
- /* Report that we are now reindexing relations */
+ /* Report that we are now reindexing relations XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_REBUILD_INDEX);
reindex_relation(OIDOldHeap, reindex_flags, &reindex_params);
- /* Report that we are now doing clean up */
+ /* Report that we are now doing clean up XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_FINAL_CLEANUP);
@@ -1633,9 +1639,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
/* Cluster each relation in a separate transaction */
static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
{
ListCell *lc;
+ off_t childnum = 0;
/* Commit to get out of starting transaction */
PopActiveSnapshot();
@@ -1658,6 +1665,11 @@ cluster_multiple_rels(List *rvs, int options)
cluster_rel(rvtc->tableOid, rvtc->indexOid,
&cluster_params);
+ if (ispartitioned)
+ pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+ childnum++);
+
+
PopActiveSnapshot();
CommitTransactionCommand();
}
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
#define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS 5
#define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED 6
#define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT 7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL 17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE 18
/* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
#define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP 1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b632d9f2ea..d1fea532bc 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1941,7 +1941,9 @@ pg_stat_progress_cluster| SELECT s.pid,
s.param5 AS heap_tuples_written,
s.param6 AS heap_blks_total,
s.param7 AS heap_blks_scanned,
- s.param8 AS index_rebuild_count
+ s.param8 AS index_rebuild_count,
+ s.param18 AS partitions_total,
+ s.param19 AS partitions_done
FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
LEFT JOIN pg_database d ON ((s.datid = d.oid)));
pg_stat_progress_copy| SELECT s.pid,
--
2.17.0
--O5XBE6gyVG5Rl6Rj
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v8-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"
^ permalink raw reply [nested|flat] 73+ messages in thread
* [PATCH v10 3/8] f! progress reporting..
@ 2021-02-09 02:45 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 73+ messages in thread
From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)
This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.
This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
doc/src/sgml/monitoring.sgml | 21 +++++++++++
src/backend/catalog/system_views.sql | 4 +-
src/backend/commands/cluster.c | 55 ++++++++++++++++------------
src/backend/commands/vacuum.c | 5 +++
src/include/commands/progress.h | 3 ++
src/test/regress/expected/rules.out | 4 +-
6 files changed, 67 insertions(+), 25 deletions(-)
diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml
index 56018745c8..28493c7931 100644
--- a/doc/src/sgml/monitoring.sgml
+++ b/doc/src/sgml/monitoring.sgml
@@ -6293,6 +6293,27 @@ SELECT pg_stat_get_backend_pid(s.backendid) AS pid,
is <literal>rebuilding index</literal>.
</para></entry>
</row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>total_partitions</structfield> <type>bigint</type>
+ </para>
+ When clustering a partitioned table, this column is set to the total
+ number of partitions to be clustered.
+ <para>
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>done_partitions</structfield> <type>bigint</type>
+ </para>
+ <para>
+ When clustering a partitioned table, this column is set to the number
+ of partitions which have been clustered.
+ </para></entry>
+ </row>
+
</tbody>
</tgroup>
</table>
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index 5f2541d316..0256809f06 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1142,7 +1142,9 @@ CREATE VIEW pg_stat_progress_cluster AS
S.param5 AS heap_tuples_written,
S.param6 AS heap_blks_total,
S.param7 AS heap_blks_scanned,
- S.param8 AS index_rebuild_count
+ S.param8 AS index_rebuild_count,
+ S.param18 AS partitions_total,
+ S.param19 AS partitions_done
FROM pg_stat_get_progress_info('CLUSTER') AS S
LEFT JOIN pg_database D ON S.datid = D.oid;
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 0c08ac56dc..7d8d1d8a69 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
static List *get_tables_to_cluster(MemoryContext cluster_context);
static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
/*---------------------------------------------------------------------------
@@ -188,16 +188,26 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
/* close relation, keep lock till commit */
table_close(rel, ShareUpdateExclusiveLock);
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
{
/* Do the job. */
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_CLUSTER);
cluster_rel(tableOid, indexOid, ¶ms);
+ pgstat_progress_end_command();
}
else
{
List *rvs;
MemoryContext cluster_context;
+ int64 progress_val[2];
+ const int progress_index[2] = {
+ PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+ };
+
/* Refuse to hold strong locks in a user transaction */
PreventInTransactionBlock(isTopLevel, "CLUSTER");
@@ -206,7 +216,15 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
ALLOCSET_DEFAULT_SIZES);
rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
- cluster_multiple_rels(rvs, params.options);
+
+ /* Report command and number of partitions */
+ progress_val[0] = PROGRESS_CLUSTER_COMMAND_CLUSTER;
+ /* XXX: rvs includes the parent, which is not a "partition" */
+ progress_val[1] = list_length(rvs);
+
+ pgstat_progress_update_multi_param(2, progress_index, progress_val);
+ cluster_multiple_rels(rvs, params.options, true);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -245,7 +263,11 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
* cluster_context.
*/
rvs = get_tables_to_cluster(cluster_context);
- cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_CLUSTER);
+ cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -282,14 +304,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* Check for user-requested abort. */
CHECK_FOR_INTERRUPTS();
- pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
- if (OidIsValid(indexOid))
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_CLUSTER);
- else
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
/*
* We grab exclusive access to the target rel and index for the duration
* of the transaction. (This is redundant for the single-transaction
@@ -300,10 +314,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* If the table has gone away, we can skip processing it */
if (!OldHeap)
- {
- pgstat_progress_end_command();
return;
- }
/*
* Since we may open a new transaction for each relation, we have to check
@@ -319,7 +330,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!pg_class_ownercheck(tableOid, GetUserId()))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -334,7 +344,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (RELATION_IS_OTHER_TEMP(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -346,7 +355,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -357,7 +365,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!get_index_isclustered(indexOid))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
}
@@ -416,7 +423,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!RelationIsPopulated(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -424,7 +430,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -440,8 +445,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
rebuild_relation(OldHeap, indexOid, verbose);
/* NB: rebuild_relation does table_close() on OldHeap */
-
- pgstat_progress_end_command();
}
/*
@@ -1632,9 +1635,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
/* Cluster each relation in a separate transaction */
static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
{
ListCell *lc;
+ off_t childnum = 0;
/* Commit to get out of starting transaction */
PopActiveSnapshot();
@@ -1657,6 +1661,11 @@ cluster_multiple_rels(List *rvs, int options)
cluster_rel(rvtc->tableOid, rvtc->indexOid,
&cluster_params);
+ if (ispartitioned)
+ pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+ ++childnum);
+
+
PopActiveSnapshot();
CommitTransactionCommand();
}
diff --git a/src/backend/commands/vacuum.c b/src/backend/commands/vacuum.c
index 662aff04b4..1d8a7ca774 100644
--- a/src/backend/commands/vacuum.c
+++ b/src/backend/commands/vacuum.c
@@ -37,6 +37,7 @@
#include "catalog/pg_namespace.h"
#include "commands/cluster.h"
#include "commands/defrem.h"
+#include "commands/progress.h"
#include "commands/vacuum.h"
#include "miscadmin.h"
#include "nodes/makefuncs.h"
@@ -1937,7 +1938,11 @@ vacuum_rel(Oid relid, RangeVar *relation, VacuumParams *params)
cluster_params.options |= CLUOPT_VERBOSE;
/* VACUUM FULL is now a variant of CLUSTER; see cluster.c */
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, relid);
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
cluster_rel(relid, InvalidOid, &cluster_params);
+ pgstat_progress_end_command();
}
else
table_relation_vacuum(onerel, params, vac_strategy);
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index c6b139d57d..d85781ae0b 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
#define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS 5
#define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED 6
#define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT 7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL 17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE 18
/* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
#define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP 1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index 9b59a7b4a5..a8bcc599ac 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1943,7 +1943,9 @@ pg_stat_progress_cluster| SELECT s.pid,
s.param5 AS heap_tuples_written,
s.param6 AS heap_blks_total,
s.param7 AS heap_blks_scanned,
- s.param8 AS index_rebuild_count
+ s.param8 AS index_rebuild_count,
+ s.param18 AS partitions_total,
+ s.param19 AS partitions_done
FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
LEFT JOIN pg_database d ON ((s.datid = d.oid)));
pg_stat_progress_copy| SELECT s.pid,
--
2.17.0
--mvpLiMfbWzRoNl4x
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v10-0004-Propagate-changes-to-indisclustered-to-child-par.patch"
^ permalink raw reply [nested|flat] 73+ messages in thread
* [PATCH v8 3/8] f! progress reporting..
@ 2021-02-09 02:45 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 73+ messages in thread
From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)
This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.
This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
src/backend/catalog/system_views.sql | 4 +-
src/backend/commands/cluster.c | 64 +++++++++++++++++-----------
src/include/commands/progress.h | 3 ++
src/test/regress/expected/rules.out | 4 +-
4 files changed, 47 insertions(+), 28 deletions(-)
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fa58afd9d7..9a3cb5f935 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1069,7 +1069,9 @@ CREATE VIEW pg_stat_progress_cluster AS
S.param5 AS heap_tuples_written,
S.param6 AS heap_blks_total,
S.param7 AS heap_blks_scanned,
- S.param8 AS index_rebuild_count
+ S.param8 AS index_rebuild_count,
+ S.param18 AS partitions_total,
+ S.param19 AS partitions_done
FROM pg_stat_get_progress_info('CLUSTER') AS S
LEFT JOIN pg_database D ON S.datid = D.oid;
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 9b6673867c..cb4fc350c6 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
static List *get_tables_to_cluster(MemoryContext cluster_context);
static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
/*---------------------------------------------------------------------------
@@ -188,16 +188,28 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
/* close relation, keep lock till commit */
table_close(rel, ShareUpdateExclusiveLock);
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
{
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ OidIsValid(indexOid) ? PROGRESS_CLUSTER_COMMAND_CLUSTER :
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
+
/* Do the job. */
cluster_rel(tableOid, indexOid, ¶ms);
+ pgstat_progress_end_command();
}
else
{
List *rvs;
MemoryContext cluster_context;
+ int64 progress_val[2];
+ const int progress_index[] = {
+ PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+ };
+
/* Refuse to hold strong locks in a user transaction */
PreventInTransactionBlock(isTopLevel, "CLUSTER");
@@ -206,7 +218,20 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
ALLOCSET_DEFAULT_SIZES);
rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
- cluster_multiple_rels(rvs, params.options);
+
+ /* Report vacuum full or cluster and number of partitions */
+ progress_val[0] = OidIsValid(indexOid) ?
+ PROGRESS_CLUSTER_COMMAND_CLUSTER :
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL;
+ progress_val[1] = list_length(rvs);
+// This is currently showing the total number of *tables*, including the
+// parent, and intermediate partitions, so the column should be renamed, or
+// we should count the number of leaf partitions
+
+ pgstat_progress_update_multi_param(2, progress_index, progress_val);
+
+ cluster_multiple_rels(rvs, params.options, true);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -245,7 +270,7 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
* cluster_context.
*/
rvs = get_tables_to_cluster(cluster_context);
- cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+ cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -282,14 +307,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* Check for user-requested abort. */
CHECK_FOR_INTERRUPTS();
- pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
- if (OidIsValid(indexOid))
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_CLUSTER);
- else
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
/*
* We grab exclusive access to the target rel and index for the duration
* of the transaction. (This is redundant for the single-transaction
@@ -300,10 +317,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* If the table has gone away, we can skip processing it */
if (!OldHeap)
- {
- pgstat_progress_end_command();
return;
- }
/*
* Since we may open a new transaction for each relation, we have to check
@@ -319,7 +333,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!pg_class_ownercheck(tableOid, GetUserId()))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -334,7 +347,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (RELATION_IS_OTHER_TEMP(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -346,7 +358,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -357,7 +368,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!get_index_isclustered(indexOid))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
}
@@ -416,7 +426,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!RelationIsPopulated(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -424,7 +433,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -440,8 +448,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
rebuild_relation(OldHeap, indexOid, verbose);
/* NB: rebuild_relation does table_close() on OldHeap */
-
- pgstat_progress_end_command();
}
/*
@@ -1366,7 +1372,7 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap,
ReindexParams reindex_params = {0};
int i;
- /* Report that we are now swapping relation files */
+ /* Report that we are now swapping relation files XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_SWAP_REL_FILES);
@@ -1417,13 +1423,13 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap,
else if (newrelpersistence == RELPERSISTENCE_PERMANENT)
reindex_flags |= REINDEX_REL_FORCE_INDEXES_PERMANENT;
- /* Report that we are now reindexing relations */
+ /* Report that we are now reindexing relations XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_REBUILD_INDEX);
reindex_relation(OIDOldHeap, reindex_flags, &reindex_params);
- /* Report that we are now doing clean up */
+ /* Report that we are now doing clean up XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_FINAL_CLEANUP);
@@ -1633,9 +1639,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
/* Cluster each relation in a separate transaction */
static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
{
ListCell *lc;
+ off_t childnum = 0;
/* Commit to get out of starting transaction */
PopActiveSnapshot();
@@ -1658,6 +1665,11 @@ cluster_multiple_rels(List *rvs, int options)
cluster_rel(rvtc->tableOid, rvtc->indexOid,
&cluster_params);
+ if (ispartitioned)
+ pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+ childnum++);
+
+
PopActiveSnapshot();
CommitTransactionCommand();
}
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
#define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS 5
#define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED 6
#define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT 7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL 17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE 18
/* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
#define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP 1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b632d9f2ea..d1fea532bc 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1941,7 +1941,9 @@ pg_stat_progress_cluster| SELECT s.pid,
s.param5 AS heap_tuples_written,
s.param6 AS heap_blks_total,
s.param7 AS heap_blks_scanned,
- s.param8 AS index_rebuild_count
+ s.param8 AS index_rebuild_count,
+ s.param18 AS partitions_total,
+ s.param19 AS partitions_done
FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
LEFT JOIN pg_database d ON ((s.datid = d.oid)));
pg_stat_progress_copy| SELECT s.pid,
--
2.17.0
--O5XBE6gyVG5Rl6Rj
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v8-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"
^ permalink raw reply [nested|flat] 73+ messages in thread
* [PATCH v13 02/18] f! progress reporting
@ 2021-02-15 00:31 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 73+ messages in thread
From: Justin Pryzby @ 2021-02-15 00:31 UTC (permalink / raw)
---
src/backend/commands/indexcmds.c | 33 +++++++-------------------------
1 file changed, 7 insertions(+), 26 deletions(-)
diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 4ac1dacd7d..d6567ec231 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1638,40 +1638,20 @@ reindex_invalid_child_indexes(Oid indexRelationId)
.options = REINDEXOPT_CONCURRENTLY
};
- MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX",
- ALLOCSET_DEFAULT_SIZES);
- MemoryContext oldcontext;
- List *childs = find_inheritance_children(indexRelationId, ShareLock);
- List *partitions = NIL;
-
PreventInTransactionBlock(true, "REINDEX INDEX");
- foreach (lc, childs)
+ foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
{
Oid partoid = lfirst_oid(lc);
- /* XXX: need to retrofit progress reporting into it */
- // pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
- // npart++);
-
- if (get_index_isvalid(partoid) ||
- !RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
- continue;
+ if (!get_index_isvalid(partoid) &&
+ RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
+ ReindexRelationConcurrently(partoid, ¶ms);
- /* Save partition OID */
- oldcontext = MemoryContextSwitchTo(ind_context);
- partitions = lappend_oid(partitions, partoid);
- MemoryContextSwitchTo(oldcontext);
+ pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
+ npart++);
}
- /*
- * Process each partition listed in a separate transaction. Note that
- * this commits and then starts a new transaction immediately.
- * XXX: since this is done in 2*N transactions, it could just as well
- * call ReindexRelationConcurrently directly
- */
- ReindexMultipleInternal(partitions, ¶ms);
-
/*
* CIC needs to mark a partitioned index as VALID, which itself
* requires setting READY, which is unset for CIC (even though
@@ -1680,6 +1660,7 @@ reindex_invalid_child_indexes(Oid indexRelationId)
* partitions.
* See also: validatePartitionedIndex().
*/
+ CommandCounterIncrement();
index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
CommandCounterIncrement();
index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
--
2.17.0
--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13-0003-WIP-Add-SKIPVALID-flag-for-more-integration.patch"
^ permalink raw reply [nested|flat] 73+ messages in thread
* [PATCH v13 02/18] f! progress reporting
@ 2021-02-15 00:31 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 73+ messages in thread
From: Justin Pryzby @ 2021-02-15 00:31 UTC (permalink / raw)
---
src/backend/commands/indexcmds.c | 33 +++++++-------------------------
1 file changed, 7 insertions(+), 26 deletions(-)
diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 4ac1dacd7d..d6567ec231 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1638,40 +1638,20 @@ reindex_invalid_child_indexes(Oid indexRelationId)
.options = REINDEXOPT_CONCURRENTLY
};
- MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX",
- ALLOCSET_DEFAULT_SIZES);
- MemoryContext oldcontext;
- List *childs = find_inheritance_children(indexRelationId, ShareLock);
- List *partitions = NIL;
-
PreventInTransactionBlock(true, "REINDEX INDEX");
- foreach (lc, childs)
+ foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
{
Oid partoid = lfirst_oid(lc);
- /* XXX: need to retrofit progress reporting into it */
- // pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
- // npart++);
-
- if (get_index_isvalid(partoid) ||
- !RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
- continue;
+ if (!get_index_isvalid(partoid) &&
+ RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
+ ReindexRelationConcurrently(partoid, ¶ms);
- /* Save partition OID */
- oldcontext = MemoryContextSwitchTo(ind_context);
- partitions = lappend_oid(partitions, partoid);
- MemoryContextSwitchTo(oldcontext);
+ pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
+ npart++);
}
- /*
- * Process each partition listed in a separate transaction. Note that
- * this commits and then starts a new transaction immediately.
- * XXX: since this is done in 2*N transactions, it could just as well
- * call ReindexRelationConcurrently directly
- */
- ReindexMultipleInternal(partitions, ¶ms);
-
/*
* CIC needs to mark a partitioned index as VALID, which itself
* requires setting READY, which is unset for CIC (even though
@@ -1680,6 +1660,7 @@ reindex_invalid_child_indexes(Oid indexRelationId)
* partitions.
* See also: validatePartitionedIndex().
*/
+ CommandCounterIncrement();
index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
CommandCounterIncrement();
index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
--
2.17.0
--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13-0003-WIP-Add-SKIPVALID-flag-for-more-integration.patch"
^ permalink raw reply [nested|flat] 73+ messages in thread
* [PATCH v13 02/18] f! progress reporting
@ 2021-02-15 00:31 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 73+ messages in thread
From: Justin Pryzby @ 2021-02-15 00:31 UTC (permalink / raw)
---
src/backend/commands/indexcmds.c | 33 +++++++-------------------------
1 file changed, 7 insertions(+), 26 deletions(-)
diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 4ac1dacd7d..d6567ec231 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1638,40 +1638,20 @@ reindex_invalid_child_indexes(Oid indexRelationId)
.options = REINDEXOPT_CONCURRENTLY
};
- MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX",
- ALLOCSET_DEFAULT_SIZES);
- MemoryContext oldcontext;
- List *childs = find_inheritance_children(indexRelationId, ShareLock);
- List *partitions = NIL;
-
PreventInTransactionBlock(true, "REINDEX INDEX");
- foreach (lc, childs)
+ foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
{
Oid partoid = lfirst_oid(lc);
- /* XXX: need to retrofit progress reporting into it */
- // pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
- // npart++);
-
- if (get_index_isvalid(partoid) ||
- !RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
- continue;
+ if (!get_index_isvalid(partoid) &&
+ RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
+ ReindexRelationConcurrently(partoid, ¶ms);
- /* Save partition OID */
- oldcontext = MemoryContextSwitchTo(ind_context);
- partitions = lappend_oid(partitions, partoid);
- MemoryContextSwitchTo(oldcontext);
+ pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
+ npart++);
}
- /*
- * Process each partition listed in a separate transaction. Note that
- * this commits and then starts a new transaction immediately.
- * XXX: since this is done in 2*N transactions, it could just as well
- * call ReindexRelationConcurrently directly
- */
- ReindexMultipleInternal(partitions, ¶ms);
-
/*
* CIC needs to mark a partitioned index as VALID, which itself
* requires setting READY, which is unset for CIC (even though
@@ -1680,6 +1660,7 @@ reindex_invalid_child_indexes(Oid indexRelationId)
* partitions.
* See also: validatePartitionedIndex().
*/
+ CommandCounterIncrement();
index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
CommandCounterIncrement();
index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
--
2.17.0
--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13-0003-WIP-Add-SKIPVALID-flag-for-more-integration.patch"
^ permalink raw reply [nested|flat] 73+ messages in thread
* [PATCH v13 02/18] f! progress reporting
@ 2021-02-15 00:31 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 73+ messages in thread
From: Justin Pryzby @ 2021-02-15 00:31 UTC (permalink / raw)
---
src/backend/commands/indexcmds.c | 33 +++++++-------------------------
1 file changed, 7 insertions(+), 26 deletions(-)
diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 4ac1dacd7d..d6567ec231 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1638,40 +1638,20 @@ reindex_invalid_child_indexes(Oid indexRelationId)
.options = REINDEXOPT_CONCURRENTLY
};
- MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX",
- ALLOCSET_DEFAULT_SIZES);
- MemoryContext oldcontext;
- List *childs = find_inheritance_children(indexRelationId, ShareLock);
- List *partitions = NIL;
-
PreventInTransactionBlock(true, "REINDEX INDEX");
- foreach (lc, childs)
+ foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
{
Oid partoid = lfirst_oid(lc);
- /* XXX: need to retrofit progress reporting into it */
- // pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
- // npart++);
-
- if (get_index_isvalid(partoid) ||
- !RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
- continue;
+ if (!get_index_isvalid(partoid) &&
+ RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
+ ReindexRelationConcurrently(partoid, ¶ms);
- /* Save partition OID */
- oldcontext = MemoryContextSwitchTo(ind_context);
- partitions = lappend_oid(partitions, partoid);
- MemoryContextSwitchTo(oldcontext);
+ pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
+ npart++);
}
- /*
- * Process each partition listed in a separate transaction. Note that
- * this commits and then starts a new transaction immediately.
- * XXX: since this is done in 2*N transactions, it could just as well
- * call ReindexRelationConcurrently directly
- */
- ReindexMultipleInternal(partitions, ¶ms);
-
/*
* CIC needs to mark a partitioned index as VALID, which itself
* requires setting READY, which is unset for CIC (even though
@@ -1680,6 +1660,7 @@ reindex_invalid_child_indexes(Oid indexRelationId)
* partitions.
* See also: validatePartitionedIndex().
*/
+ CommandCounterIncrement();
index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
CommandCounterIncrement();
index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
--
2.17.0
--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13-0003-WIP-Add-SKIPVALID-flag-for-more-integration.patch"
^ permalink raw reply [nested|flat] 73+ messages in thread
* [PATCH v13 02/18] f! progress reporting
@ 2021-02-15 00:31 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 73+ messages in thread
From: Justin Pryzby @ 2021-02-15 00:31 UTC (permalink / raw)
---
src/backend/commands/indexcmds.c | 33 +++++++-------------------------
1 file changed, 7 insertions(+), 26 deletions(-)
diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 4ac1dacd7d..d6567ec231 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1638,40 +1638,20 @@ reindex_invalid_child_indexes(Oid indexRelationId)
.options = REINDEXOPT_CONCURRENTLY
};
- MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX",
- ALLOCSET_DEFAULT_SIZES);
- MemoryContext oldcontext;
- List *childs = find_inheritance_children(indexRelationId, ShareLock);
- List *partitions = NIL;
-
PreventInTransactionBlock(true, "REINDEX INDEX");
- foreach (lc, childs)
+ foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
{
Oid partoid = lfirst_oid(lc);
- /* XXX: need to retrofit progress reporting into it */
- // pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
- // npart++);
-
- if (get_index_isvalid(partoid) ||
- !RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
- continue;
+ if (!get_index_isvalid(partoid) &&
+ RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
+ ReindexRelationConcurrently(partoid, ¶ms);
- /* Save partition OID */
- oldcontext = MemoryContextSwitchTo(ind_context);
- partitions = lappend_oid(partitions, partoid);
- MemoryContextSwitchTo(oldcontext);
+ pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
+ npart++);
}
- /*
- * Process each partition listed in a separate transaction. Note that
- * this commits and then starts a new transaction immediately.
- * XXX: since this is done in 2*N transactions, it could just as well
- * call ReindexRelationConcurrently directly
- */
- ReindexMultipleInternal(partitions, ¶ms);
-
/*
* CIC needs to mark a partitioned index as VALID, which itself
* requires setting READY, which is unset for CIC (even though
@@ -1680,6 +1660,7 @@ reindex_invalid_child_indexes(Oid indexRelationId)
* partitions.
* See also: validatePartitionedIndex().
*/
+ CommandCounterIncrement();
index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
CommandCounterIncrement();
index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
--
2.17.0
--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13-0003-WIP-Add-SKIPVALID-flag-for-more-integration.patch"
^ permalink raw reply [nested|flat] 73+ messages in thread
* [PATCH 2/5] f! progress reporting
@ 2021-02-15 00:31 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 73+ messages in thread
From: Justin Pryzby @ 2021-02-15 00:31 UTC (permalink / raw)
---
src/backend/commands/indexcmds.c | 33 +++++++-------------------------
1 file changed, 7 insertions(+), 26 deletions(-)
diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 9ab1a66971..8f4eab22eb 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1647,40 +1647,20 @@ reindex_invalid_child_indexes(Oid indexRelationId)
.options = REINDEXOPT_CONCURRENTLY
};
- MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX",
- ALLOCSET_DEFAULT_SIZES);
- MemoryContext oldcontext;
- List *childs = find_inheritance_children(indexRelationId, ShareLock);
- List *partitions = NIL;
-
PreventInTransactionBlock(true, "REINDEX INDEX");
- foreach (lc, childs)
+ foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
{
Oid partoid = lfirst_oid(lc);
- /* XXX: need to retrofit progress reporting into it */
- // pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
- // npart++);
-
- if (get_index_isvalid(partoid) ||
- !RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
- continue;
+ if (!get_index_isvalid(partoid) &&
+ RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
+ ReindexRelationConcurrently(partoid, ¶ms);
- /* Save partition OID */
- oldcontext = MemoryContextSwitchTo(ind_context);
- partitions = lappend_oid(partitions, partoid);
- MemoryContextSwitchTo(oldcontext);
+ pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
+ npart++);
}
- /*
- * Process each partition listed in a separate transaction. Note that
- * this commits and then starts a new transaction immediately.
- * XXX: since this is done in 2*N transactions, it could just as well
- * call ReindexRelationConcurrently directly
- */
- ReindexMultipleInternal(partitions, ¶ms);
-
/*
* CIC needs to mark a partitioned index as VALID, which itself
* requires setting READY, which is unset for CIC (even though
@@ -1689,6 +1669,7 @@ reindex_invalid_child_indexes(Oid indexRelationId)
* partitions.
* See also: validatePartitionedIndex().
*/
+ CommandCounterIncrement();
index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
CommandCounterIncrement();
index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
--
2.17.0
--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="0003-WIP-Add-SKIPVALID-flag-for-more-integration.patch"
^ permalink raw reply [nested|flat] 73+ messages in thread
* [PATCH v13 02/18] f! progress reporting
@ 2021-02-15 00:31 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 73+ messages in thread
From: Justin Pryzby @ 2021-02-15 00:31 UTC (permalink / raw)
---
src/backend/commands/indexcmds.c | 33 +++++++-------------------------
1 file changed, 7 insertions(+), 26 deletions(-)
diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 4ac1dacd7d..d6567ec231 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1638,40 +1638,20 @@ reindex_invalid_child_indexes(Oid indexRelationId)
.options = REINDEXOPT_CONCURRENTLY
};
- MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX",
- ALLOCSET_DEFAULT_SIZES);
- MemoryContext oldcontext;
- List *childs = find_inheritance_children(indexRelationId, ShareLock);
- List *partitions = NIL;
-
PreventInTransactionBlock(true, "REINDEX INDEX");
- foreach (lc, childs)
+ foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
{
Oid partoid = lfirst_oid(lc);
- /* XXX: need to retrofit progress reporting into it */
- // pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
- // npart++);
-
- if (get_index_isvalid(partoid) ||
- !RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
- continue;
+ if (!get_index_isvalid(partoid) &&
+ RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
+ ReindexRelationConcurrently(partoid, ¶ms);
- /* Save partition OID */
- oldcontext = MemoryContextSwitchTo(ind_context);
- partitions = lappend_oid(partitions, partoid);
- MemoryContextSwitchTo(oldcontext);
+ pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
+ npart++);
}
- /*
- * Process each partition listed in a separate transaction. Note that
- * this commits and then starts a new transaction immediately.
- * XXX: since this is done in 2*N transactions, it could just as well
- * call ReindexRelationConcurrently directly
- */
- ReindexMultipleInternal(partitions, ¶ms);
-
/*
* CIC needs to mark a partitioned index as VALID, which itself
* requires setting READY, which is unset for CIC (even though
@@ -1680,6 +1660,7 @@ reindex_invalid_child_indexes(Oid indexRelationId)
* partitions.
* See also: validatePartitionedIndex().
*/
+ CommandCounterIncrement();
index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
CommandCounterIncrement();
index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
--
2.17.0
--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13-0003-WIP-Add-SKIPVALID-flag-for-more-integration.patch"
^ permalink raw reply [nested|flat] 73+ messages in thread
* [PATCH v13 02/18] f! progress reporting
@ 2021-02-15 00:31 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 73+ messages in thread
From: Justin Pryzby @ 2021-02-15 00:31 UTC (permalink / raw)
---
src/backend/commands/indexcmds.c | 33 +++++++-------------------------
1 file changed, 7 insertions(+), 26 deletions(-)
diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 4ac1dacd7d..d6567ec231 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1638,40 +1638,20 @@ reindex_invalid_child_indexes(Oid indexRelationId)
.options = REINDEXOPT_CONCURRENTLY
};
- MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX",
- ALLOCSET_DEFAULT_SIZES);
- MemoryContext oldcontext;
- List *childs = find_inheritance_children(indexRelationId, ShareLock);
- List *partitions = NIL;
-
PreventInTransactionBlock(true, "REINDEX INDEX");
- foreach (lc, childs)
+ foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
{
Oid partoid = lfirst_oid(lc);
- /* XXX: need to retrofit progress reporting into it */
- // pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
- // npart++);
-
- if (get_index_isvalid(partoid) ||
- !RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
- continue;
+ if (!get_index_isvalid(partoid) &&
+ RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
+ ReindexRelationConcurrently(partoid, ¶ms);
- /* Save partition OID */
- oldcontext = MemoryContextSwitchTo(ind_context);
- partitions = lappend_oid(partitions, partoid);
- MemoryContextSwitchTo(oldcontext);
+ pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
+ npart++);
}
- /*
- * Process each partition listed in a separate transaction. Note that
- * this commits and then starts a new transaction immediately.
- * XXX: since this is done in 2*N transactions, it could just as well
- * call ReindexRelationConcurrently directly
- */
- ReindexMultipleInternal(partitions, ¶ms);
-
/*
* CIC needs to mark a partitioned index as VALID, which itself
* requires setting READY, which is unset for CIC (even though
@@ -1680,6 +1660,7 @@ reindex_invalid_child_indexes(Oid indexRelationId)
* partitions.
* See also: validatePartitionedIndex().
*/
+ CommandCounterIncrement();
index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
CommandCounterIncrement();
index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
--
2.17.0
--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13-0003-WIP-Add-SKIPVALID-flag-for-more-integration.patch"
^ permalink raw reply [nested|flat] 73+ messages in thread
* [PATCH v13 02/18] f! progress reporting
@ 2021-02-15 00:31 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 73+ messages in thread
From: Justin Pryzby @ 2021-02-15 00:31 UTC (permalink / raw)
---
src/backend/commands/indexcmds.c | 33 +++++++-------------------------
1 file changed, 7 insertions(+), 26 deletions(-)
diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 4ac1dacd7d..d6567ec231 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1638,40 +1638,20 @@ reindex_invalid_child_indexes(Oid indexRelationId)
.options = REINDEXOPT_CONCURRENTLY
};
- MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX",
- ALLOCSET_DEFAULT_SIZES);
- MemoryContext oldcontext;
- List *childs = find_inheritance_children(indexRelationId, ShareLock);
- List *partitions = NIL;
-
PreventInTransactionBlock(true, "REINDEX INDEX");
- foreach (lc, childs)
+ foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
{
Oid partoid = lfirst_oid(lc);
- /* XXX: need to retrofit progress reporting into it */
- // pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
- // npart++);
-
- if (get_index_isvalid(partoid) ||
- !RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
- continue;
+ if (!get_index_isvalid(partoid) &&
+ RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
+ ReindexRelationConcurrently(partoid, ¶ms);
- /* Save partition OID */
- oldcontext = MemoryContextSwitchTo(ind_context);
- partitions = lappend_oid(partitions, partoid);
- MemoryContextSwitchTo(oldcontext);
+ pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
+ npart++);
}
- /*
- * Process each partition listed in a separate transaction. Note that
- * this commits and then starts a new transaction immediately.
- * XXX: since this is done in 2*N transactions, it could just as well
- * call ReindexRelationConcurrently directly
- */
- ReindexMultipleInternal(partitions, ¶ms);
-
/*
* CIC needs to mark a partitioned index as VALID, which itself
* requires setting READY, which is unset for CIC (even though
@@ -1680,6 +1660,7 @@ reindex_invalid_child_indexes(Oid indexRelationId)
* partitions.
* See also: validatePartitionedIndex().
*/
+ CommandCounterIncrement();
index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
CommandCounterIncrement();
index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
--
2.17.0
--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13-0003-WIP-Add-SKIPVALID-flag-for-more-integration.patch"
^ permalink raw reply [nested|flat] 73+ messages in thread
* [PATCH v13 02/18] f! progress reporting
@ 2021-02-15 00:31 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 73+ messages in thread
From: Justin Pryzby @ 2021-02-15 00:31 UTC (permalink / raw)
---
src/backend/commands/indexcmds.c | 33 +++++++-------------------------
1 file changed, 7 insertions(+), 26 deletions(-)
diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 4ac1dacd7d..d6567ec231 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1638,40 +1638,20 @@ reindex_invalid_child_indexes(Oid indexRelationId)
.options = REINDEXOPT_CONCURRENTLY
};
- MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX",
- ALLOCSET_DEFAULT_SIZES);
- MemoryContext oldcontext;
- List *childs = find_inheritance_children(indexRelationId, ShareLock);
- List *partitions = NIL;
-
PreventInTransactionBlock(true, "REINDEX INDEX");
- foreach (lc, childs)
+ foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
{
Oid partoid = lfirst_oid(lc);
- /* XXX: need to retrofit progress reporting into it */
- // pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
- // npart++);
-
- if (get_index_isvalid(partoid) ||
- !RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
- continue;
+ if (!get_index_isvalid(partoid) &&
+ RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
+ ReindexRelationConcurrently(partoid, ¶ms);
- /* Save partition OID */
- oldcontext = MemoryContextSwitchTo(ind_context);
- partitions = lappend_oid(partitions, partoid);
- MemoryContextSwitchTo(oldcontext);
+ pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
+ npart++);
}
- /*
- * Process each partition listed in a separate transaction. Note that
- * this commits and then starts a new transaction immediately.
- * XXX: since this is done in 2*N transactions, it could just as well
- * call ReindexRelationConcurrently directly
- */
- ReindexMultipleInternal(partitions, ¶ms);
-
/*
* CIC needs to mark a partitioned index as VALID, which itself
* requires setting READY, which is unset for CIC (even though
@@ -1680,6 +1660,7 @@ reindex_invalid_child_indexes(Oid indexRelationId)
* partitions.
* See also: validatePartitionedIndex().
*/
+ CommandCounterIncrement();
index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
CommandCounterIncrement();
index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
--
2.17.0
--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13-0003-WIP-Add-SKIPVALID-flag-for-more-integration.patch"
^ permalink raw reply [nested|flat] 73+ messages in thread
* [PATCH v13 02/18] f! progress reporting
@ 2021-02-15 00:31 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 73+ messages in thread
From: Justin Pryzby @ 2021-02-15 00:31 UTC (permalink / raw)
---
src/backend/commands/indexcmds.c | 33 +++++++-------------------------
1 file changed, 7 insertions(+), 26 deletions(-)
diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 4ac1dacd7d..d6567ec231 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1638,40 +1638,20 @@ reindex_invalid_child_indexes(Oid indexRelationId)
.options = REINDEXOPT_CONCURRENTLY
};
- MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX",
- ALLOCSET_DEFAULT_SIZES);
- MemoryContext oldcontext;
- List *childs = find_inheritance_children(indexRelationId, ShareLock);
- List *partitions = NIL;
-
PreventInTransactionBlock(true, "REINDEX INDEX");
- foreach (lc, childs)
+ foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
{
Oid partoid = lfirst_oid(lc);
- /* XXX: need to retrofit progress reporting into it */
- // pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
- // npart++);
-
- if (get_index_isvalid(partoid) ||
- !RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
- continue;
+ if (!get_index_isvalid(partoid) &&
+ RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
+ ReindexRelationConcurrently(partoid, ¶ms);
- /* Save partition OID */
- oldcontext = MemoryContextSwitchTo(ind_context);
- partitions = lappend_oid(partitions, partoid);
- MemoryContextSwitchTo(oldcontext);
+ pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
+ npart++);
}
- /*
- * Process each partition listed in a separate transaction. Note that
- * this commits and then starts a new transaction immediately.
- * XXX: since this is done in 2*N transactions, it could just as well
- * call ReindexRelationConcurrently directly
- */
- ReindexMultipleInternal(partitions, ¶ms);
-
/*
* CIC needs to mark a partitioned index as VALID, which itself
* requires setting READY, which is unset for CIC (even though
@@ -1680,6 +1660,7 @@ reindex_invalid_child_indexes(Oid indexRelationId)
* partitions.
* See also: validatePartitionedIndex().
*/
+ CommandCounterIncrement();
index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
CommandCounterIncrement();
index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
--
2.17.0
--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13-0003-WIP-Add-SKIPVALID-flag-for-more-integration.patch"
^ permalink raw reply [nested|flat] 73+ messages in thread
* [PATCH v13 02/18] f! progress reporting
@ 2021-02-15 00:31 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 73+ messages in thread
From: Justin Pryzby @ 2021-02-15 00:31 UTC (permalink / raw)
---
src/backend/commands/indexcmds.c | 33 +++++++-------------------------
1 file changed, 7 insertions(+), 26 deletions(-)
diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 4ac1dacd7d..d6567ec231 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1638,40 +1638,20 @@ reindex_invalid_child_indexes(Oid indexRelationId)
.options = REINDEXOPT_CONCURRENTLY
};
- MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX",
- ALLOCSET_DEFAULT_SIZES);
- MemoryContext oldcontext;
- List *childs = find_inheritance_children(indexRelationId, ShareLock);
- List *partitions = NIL;
-
PreventInTransactionBlock(true, "REINDEX INDEX");
- foreach (lc, childs)
+ foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
{
Oid partoid = lfirst_oid(lc);
- /* XXX: need to retrofit progress reporting into it */
- // pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
- // npart++);
-
- if (get_index_isvalid(partoid) ||
- !RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
- continue;
+ if (!get_index_isvalid(partoid) &&
+ RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
+ ReindexRelationConcurrently(partoid, ¶ms);
- /* Save partition OID */
- oldcontext = MemoryContextSwitchTo(ind_context);
- partitions = lappend_oid(partitions, partoid);
- MemoryContextSwitchTo(oldcontext);
+ pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
+ npart++);
}
- /*
- * Process each partition listed in a separate transaction. Note that
- * this commits and then starts a new transaction immediately.
- * XXX: since this is done in 2*N transactions, it could just as well
- * call ReindexRelationConcurrently directly
- */
- ReindexMultipleInternal(partitions, ¶ms);
-
/*
* CIC needs to mark a partitioned index as VALID, which itself
* requires setting READY, which is unset for CIC (even though
@@ -1680,6 +1660,7 @@ reindex_invalid_child_indexes(Oid indexRelationId)
* partitions.
* See also: validatePartitionedIndex().
*/
+ CommandCounterIncrement();
index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
CommandCounterIncrement();
index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
--
2.17.0
--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13-0003-WIP-Add-SKIPVALID-flag-for-more-integration.patch"
^ permalink raw reply [nested|flat] 73+ messages in thread
* [PATCH v13 02/18] f! progress reporting
@ 2021-02-15 00:31 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 73+ messages in thread
From: Justin Pryzby @ 2021-02-15 00:31 UTC (permalink / raw)
---
src/backend/commands/indexcmds.c | 33 +++++++-------------------------
1 file changed, 7 insertions(+), 26 deletions(-)
diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 4ac1dacd7d..d6567ec231 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1638,40 +1638,20 @@ reindex_invalid_child_indexes(Oid indexRelationId)
.options = REINDEXOPT_CONCURRENTLY
};
- MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX",
- ALLOCSET_DEFAULT_SIZES);
- MemoryContext oldcontext;
- List *childs = find_inheritance_children(indexRelationId, ShareLock);
- List *partitions = NIL;
-
PreventInTransactionBlock(true, "REINDEX INDEX");
- foreach (lc, childs)
+ foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
{
Oid partoid = lfirst_oid(lc);
- /* XXX: need to retrofit progress reporting into it */
- // pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
- // npart++);
-
- if (get_index_isvalid(partoid) ||
- !RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
- continue;
+ if (!get_index_isvalid(partoid) &&
+ RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
+ ReindexRelationConcurrently(partoid, ¶ms);
- /* Save partition OID */
- oldcontext = MemoryContextSwitchTo(ind_context);
- partitions = lappend_oid(partitions, partoid);
- MemoryContextSwitchTo(oldcontext);
+ pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
+ npart++);
}
- /*
- * Process each partition listed in a separate transaction. Note that
- * this commits and then starts a new transaction immediately.
- * XXX: since this is done in 2*N transactions, it could just as well
- * call ReindexRelationConcurrently directly
- */
- ReindexMultipleInternal(partitions, ¶ms);
-
/*
* CIC needs to mark a partitioned index as VALID, which itself
* requires setting READY, which is unset for CIC (even though
@@ -1680,6 +1660,7 @@ reindex_invalid_child_indexes(Oid indexRelationId)
* partitions.
* See also: validatePartitionedIndex().
*/
+ CommandCounterIncrement();
index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
CommandCounterIncrement();
index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
--
2.17.0
--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13-0003-WIP-Add-SKIPVALID-flag-for-more-integration.patch"
^ permalink raw reply [nested|flat] 73+ messages in thread
* [PATCH v13 02/18] f! progress reporting
@ 2021-02-15 00:31 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 73+ messages in thread
From: Justin Pryzby @ 2021-02-15 00:31 UTC (permalink / raw)
---
src/backend/commands/indexcmds.c | 33 +++++++-------------------------
1 file changed, 7 insertions(+), 26 deletions(-)
diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 4ac1dacd7d..d6567ec231 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1638,40 +1638,20 @@ reindex_invalid_child_indexes(Oid indexRelationId)
.options = REINDEXOPT_CONCURRENTLY
};
- MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX",
- ALLOCSET_DEFAULT_SIZES);
- MemoryContext oldcontext;
- List *childs = find_inheritance_children(indexRelationId, ShareLock);
- List *partitions = NIL;
-
PreventInTransactionBlock(true, "REINDEX INDEX");
- foreach (lc, childs)
+ foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
{
Oid partoid = lfirst_oid(lc);
- /* XXX: need to retrofit progress reporting into it */
- // pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
- // npart++);
-
- if (get_index_isvalid(partoid) ||
- !RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
- continue;
+ if (!get_index_isvalid(partoid) &&
+ RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
+ ReindexRelationConcurrently(partoid, ¶ms);
- /* Save partition OID */
- oldcontext = MemoryContextSwitchTo(ind_context);
- partitions = lappend_oid(partitions, partoid);
- MemoryContextSwitchTo(oldcontext);
+ pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
+ npart++);
}
- /*
- * Process each partition listed in a separate transaction. Note that
- * this commits and then starts a new transaction immediately.
- * XXX: since this is done in 2*N transactions, it could just as well
- * call ReindexRelationConcurrently directly
- */
- ReindexMultipleInternal(partitions, ¶ms);
-
/*
* CIC needs to mark a partitioned index as VALID, which itself
* requires setting READY, which is unset for CIC (even though
@@ -1680,6 +1660,7 @@ reindex_invalid_child_indexes(Oid indexRelationId)
* partitions.
* See also: validatePartitionedIndex().
*/
+ CommandCounterIncrement();
index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
CommandCounterIncrement();
index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
--
2.17.0
--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13-0003-WIP-Add-SKIPVALID-flag-for-more-integration.patch"
^ permalink raw reply [nested|flat] 73+ messages in thread
* [PATCH v13 02/18] f! progress reporting
@ 2021-02-15 00:31 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 73+ messages in thread
From: Justin Pryzby @ 2021-02-15 00:31 UTC (permalink / raw)
---
src/backend/commands/indexcmds.c | 33 +++++++-------------------------
1 file changed, 7 insertions(+), 26 deletions(-)
diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 4ac1dacd7d..d6567ec231 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1638,40 +1638,20 @@ reindex_invalid_child_indexes(Oid indexRelationId)
.options = REINDEXOPT_CONCURRENTLY
};
- MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX",
- ALLOCSET_DEFAULT_SIZES);
- MemoryContext oldcontext;
- List *childs = find_inheritance_children(indexRelationId, ShareLock);
- List *partitions = NIL;
-
PreventInTransactionBlock(true, "REINDEX INDEX");
- foreach (lc, childs)
+ foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
{
Oid partoid = lfirst_oid(lc);
- /* XXX: need to retrofit progress reporting into it */
- // pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
- // npart++);
-
- if (get_index_isvalid(partoid) ||
- !RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
- continue;
+ if (!get_index_isvalid(partoid) &&
+ RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
+ ReindexRelationConcurrently(partoid, ¶ms);
- /* Save partition OID */
- oldcontext = MemoryContextSwitchTo(ind_context);
- partitions = lappend_oid(partitions, partoid);
- MemoryContextSwitchTo(oldcontext);
+ pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
+ npart++);
}
- /*
- * Process each partition listed in a separate transaction. Note that
- * this commits and then starts a new transaction immediately.
- * XXX: since this is done in 2*N transactions, it could just as well
- * call ReindexRelationConcurrently directly
- */
- ReindexMultipleInternal(partitions, ¶ms);
-
/*
* CIC needs to mark a partitioned index as VALID, which itself
* requires setting READY, which is unset for CIC (even though
@@ -1680,6 +1660,7 @@ reindex_invalid_child_indexes(Oid indexRelationId)
* partitions.
* See also: validatePartitionedIndex().
*/
+ CommandCounterIncrement();
index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
CommandCounterIncrement();
index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
--
2.17.0
--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13-0003-WIP-Add-SKIPVALID-flag-for-more-integration.patch"
^ permalink raw reply [nested|flat] 73+ messages in thread
* [PATCH v13 02/18] f! progress reporting
@ 2021-02-15 00:31 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 73+ messages in thread
From: Justin Pryzby @ 2021-02-15 00:31 UTC (permalink / raw)
---
src/backend/commands/indexcmds.c | 33 +++++++-------------------------
1 file changed, 7 insertions(+), 26 deletions(-)
diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 4ac1dacd7d..d6567ec231 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1638,40 +1638,20 @@ reindex_invalid_child_indexes(Oid indexRelationId)
.options = REINDEXOPT_CONCURRENTLY
};
- MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX",
- ALLOCSET_DEFAULT_SIZES);
- MemoryContext oldcontext;
- List *childs = find_inheritance_children(indexRelationId, ShareLock);
- List *partitions = NIL;
-
PreventInTransactionBlock(true, "REINDEX INDEX");
- foreach (lc, childs)
+ foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
{
Oid partoid = lfirst_oid(lc);
- /* XXX: need to retrofit progress reporting into it */
- // pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
- // npart++);
-
- if (get_index_isvalid(partoid) ||
- !RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
- continue;
+ if (!get_index_isvalid(partoid) &&
+ RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
+ ReindexRelationConcurrently(partoid, ¶ms);
- /* Save partition OID */
- oldcontext = MemoryContextSwitchTo(ind_context);
- partitions = lappend_oid(partitions, partoid);
- MemoryContextSwitchTo(oldcontext);
+ pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
+ npart++);
}
- /*
- * Process each partition listed in a separate transaction. Note that
- * this commits and then starts a new transaction immediately.
- * XXX: since this is done in 2*N transactions, it could just as well
- * call ReindexRelationConcurrently directly
- */
- ReindexMultipleInternal(partitions, ¶ms);
-
/*
* CIC needs to mark a partitioned index as VALID, which itself
* requires setting READY, which is unset for CIC (even though
@@ -1680,6 +1660,7 @@ reindex_invalid_child_indexes(Oid indexRelationId)
* partitions.
* See also: validatePartitionedIndex().
*/
+ CommandCounterIncrement();
index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
CommandCounterIncrement();
index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
--
2.17.0
--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13-0003-WIP-Add-SKIPVALID-flag-for-more-integration.patch"
^ permalink raw reply [nested|flat] 73+ messages in thread
* [PATCH v13 02/18] f! progress reporting
@ 2021-02-15 00:31 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 73+ messages in thread
From: Justin Pryzby @ 2021-02-15 00:31 UTC (permalink / raw)
---
src/backend/commands/indexcmds.c | 33 +++++++-------------------------
1 file changed, 7 insertions(+), 26 deletions(-)
diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 4ac1dacd7d..d6567ec231 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1638,40 +1638,20 @@ reindex_invalid_child_indexes(Oid indexRelationId)
.options = REINDEXOPT_CONCURRENTLY
};
- MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX",
- ALLOCSET_DEFAULT_SIZES);
- MemoryContext oldcontext;
- List *childs = find_inheritance_children(indexRelationId, ShareLock);
- List *partitions = NIL;
-
PreventInTransactionBlock(true, "REINDEX INDEX");
- foreach (lc, childs)
+ foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
{
Oid partoid = lfirst_oid(lc);
- /* XXX: need to retrofit progress reporting into it */
- // pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
- // npart++);
-
- if (get_index_isvalid(partoid) ||
- !RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
- continue;
+ if (!get_index_isvalid(partoid) &&
+ RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
+ ReindexRelationConcurrently(partoid, ¶ms);
- /* Save partition OID */
- oldcontext = MemoryContextSwitchTo(ind_context);
- partitions = lappend_oid(partitions, partoid);
- MemoryContextSwitchTo(oldcontext);
+ pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
+ npart++);
}
- /*
- * Process each partition listed in a separate transaction. Note that
- * this commits and then starts a new transaction immediately.
- * XXX: since this is done in 2*N transactions, it could just as well
- * call ReindexRelationConcurrently directly
- */
- ReindexMultipleInternal(partitions, ¶ms);
-
/*
* CIC needs to mark a partitioned index as VALID, which itself
* requires setting READY, which is unset for CIC (even though
@@ -1680,6 +1660,7 @@ reindex_invalid_child_indexes(Oid indexRelationId)
* partitions.
* See also: validatePartitionedIndex().
*/
+ CommandCounterIncrement();
index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
CommandCounterIncrement();
index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
--
2.17.0
--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13-0003-WIP-Add-SKIPVALID-flag-for-more-integration.patch"
^ permalink raw reply [nested|flat] 73+ messages in thread
* [PATCH v13 02/18] f! progress reporting
@ 2021-02-15 00:31 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 73+ messages in thread
From: Justin Pryzby @ 2021-02-15 00:31 UTC (permalink / raw)
---
src/backend/commands/indexcmds.c | 33 +++++++-------------------------
1 file changed, 7 insertions(+), 26 deletions(-)
diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 4ac1dacd7d..d6567ec231 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1638,40 +1638,20 @@ reindex_invalid_child_indexes(Oid indexRelationId)
.options = REINDEXOPT_CONCURRENTLY
};
- MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX",
- ALLOCSET_DEFAULT_SIZES);
- MemoryContext oldcontext;
- List *childs = find_inheritance_children(indexRelationId, ShareLock);
- List *partitions = NIL;
-
PreventInTransactionBlock(true, "REINDEX INDEX");
- foreach (lc, childs)
+ foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
{
Oid partoid = lfirst_oid(lc);
- /* XXX: need to retrofit progress reporting into it */
- // pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
- // npart++);
-
- if (get_index_isvalid(partoid) ||
- !RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
- continue;
+ if (!get_index_isvalid(partoid) &&
+ RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
+ ReindexRelationConcurrently(partoid, ¶ms);
- /* Save partition OID */
- oldcontext = MemoryContextSwitchTo(ind_context);
- partitions = lappend_oid(partitions, partoid);
- MemoryContextSwitchTo(oldcontext);
+ pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
+ npart++);
}
- /*
- * Process each partition listed in a separate transaction. Note that
- * this commits and then starts a new transaction immediately.
- * XXX: since this is done in 2*N transactions, it could just as well
- * call ReindexRelationConcurrently directly
- */
- ReindexMultipleInternal(partitions, ¶ms);
-
/*
* CIC needs to mark a partitioned index as VALID, which itself
* requires setting READY, which is unset for CIC (even though
@@ -1680,6 +1660,7 @@ reindex_invalid_child_indexes(Oid indexRelationId)
* partitions.
* See also: validatePartitionedIndex().
*/
+ CommandCounterIncrement();
index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
CommandCounterIncrement();
index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
--
2.17.0
--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13-0003-WIP-Add-SKIPVALID-flag-for-more-integration.patch"
^ permalink raw reply [nested|flat] 73+ messages in thread
* [PATCH v13 02/18] f! progress reporting
@ 2021-02-15 00:31 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 73+ messages in thread
From: Justin Pryzby @ 2021-02-15 00:31 UTC (permalink / raw)
---
src/backend/commands/indexcmds.c | 33 +++++++-------------------------
1 file changed, 7 insertions(+), 26 deletions(-)
diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 4ac1dacd7d..d6567ec231 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1638,40 +1638,20 @@ reindex_invalid_child_indexes(Oid indexRelationId)
.options = REINDEXOPT_CONCURRENTLY
};
- MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX",
- ALLOCSET_DEFAULT_SIZES);
- MemoryContext oldcontext;
- List *childs = find_inheritance_children(indexRelationId, ShareLock);
- List *partitions = NIL;
-
PreventInTransactionBlock(true, "REINDEX INDEX");
- foreach (lc, childs)
+ foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
{
Oid partoid = lfirst_oid(lc);
- /* XXX: need to retrofit progress reporting into it */
- // pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
- // npart++);
-
- if (get_index_isvalid(partoid) ||
- !RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
- continue;
+ if (!get_index_isvalid(partoid) &&
+ RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
+ ReindexRelationConcurrently(partoid, ¶ms);
- /* Save partition OID */
- oldcontext = MemoryContextSwitchTo(ind_context);
- partitions = lappend_oid(partitions, partoid);
- MemoryContextSwitchTo(oldcontext);
+ pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
+ npart++);
}
- /*
- * Process each partition listed in a separate transaction. Note that
- * this commits and then starts a new transaction immediately.
- * XXX: since this is done in 2*N transactions, it could just as well
- * call ReindexRelationConcurrently directly
- */
- ReindexMultipleInternal(partitions, ¶ms);
-
/*
* CIC needs to mark a partitioned index as VALID, which itself
* requires setting READY, which is unset for CIC (even though
@@ -1680,6 +1660,7 @@ reindex_invalid_child_indexes(Oid indexRelationId)
* partitions.
* See also: validatePartitionedIndex().
*/
+ CommandCounterIncrement();
index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
CommandCounterIncrement();
index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
--
2.17.0
--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13-0003-WIP-Add-SKIPVALID-flag-for-more-integration.patch"
^ permalink raw reply [nested|flat] 73+ messages in thread
* [PATCH v13 02/18] f! progress reporting
@ 2021-02-15 00:31 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 73+ messages in thread
From: Justin Pryzby @ 2021-02-15 00:31 UTC (permalink / raw)
---
src/backend/commands/indexcmds.c | 33 +++++++-------------------------
1 file changed, 7 insertions(+), 26 deletions(-)
diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 4ac1dacd7d..d6567ec231 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1638,40 +1638,20 @@ reindex_invalid_child_indexes(Oid indexRelationId)
.options = REINDEXOPT_CONCURRENTLY
};
- MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX",
- ALLOCSET_DEFAULT_SIZES);
- MemoryContext oldcontext;
- List *childs = find_inheritance_children(indexRelationId, ShareLock);
- List *partitions = NIL;
-
PreventInTransactionBlock(true, "REINDEX INDEX");
- foreach (lc, childs)
+ foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
{
Oid partoid = lfirst_oid(lc);
- /* XXX: need to retrofit progress reporting into it */
- // pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
- // npart++);
-
- if (get_index_isvalid(partoid) ||
- !RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
- continue;
+ if (!get_index_isvalid(partoid) &&
+ RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
+ ReindexRelationConcurrently(partoid, ¶ms);
- /* Save partition OID */
- oldcontext = MemoryContextSwitchTo(ind_context);
- partitions = lappend_oid(partitions, partoid);
- MemoryContextSwitchTo(oldcontext);
+ pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
+ npart++);
}
- /*
- * Process each partition listed in a separate transaction. Note that
- * this commits and then starts a new transaction immediately.
- * XXX: since this is done in 2*N transactions, it could just as well
- * call ReindexRelationConcurrently directly
- */
- ReindexMultipleInternal(partitions, ¶ms);
-
/*
* CIC needs to mark a partitioned index as VALID, which itself
* requires setting READY, which is unset for CIC (even though
@@ -1680,6 +1660,7 @@ reindex_invalid_child_indexes(Oid indexRelationId)
* partitions.
* See also: validatePartitionedIndex().
*/
+ CommandCounterIncrement();
index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
CommandCounterIncrement();
index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
--
2.17.0
--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13-0003-WIP-Add-SKIPVALID-flag-for-more-integration.patch"
^ permalink raw reply [nested|flat] 73+ messages in thread
* [PATCH v13 02/18] f! progress reporting
@ 2021-02-15 00:31 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 73+ messages in thread
From: Justin Pryzby @ 2021-02-15 00:31 UTC (permalink / raw)
---
src/backend/commands/indexcmds.c | 33 +++++++-------------------------
1 file changed, 7 insertions(+), 26 deletions(-)
diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 4ac1dacd7d..d6567ec231 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1638,40 +1638,20 @@ reindex_invalid_child_indexes(Oid indexRelationId)
.options = REINDEXOPT_CONCURRENTLY
};
- MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX",
- ALLOCSET_DEFAULT_SIZES);
- MemoryContext oldcontext;
- List *childs = find_inheritance_children(indexRelationId, ShareLock);
- List *partitions = NIL;
-
PreventInTransactionBlock(true, "REINDEX INDEX");
- foreach (lc, childs)
+ foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
{
Oid partoid = lfirst_oid(lc);
- /* XXX: need to retrofit progress reporting into it */
- // pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
- // npart++);
-
- if (get_index_isvalid(partoid) ||
- !RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
- continue;
+ if (!get_index_isvalid(partoid) &&
+ RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
+ ReindexRelationConcurrently(partoid, ¶ms);
- /* Save partition OID */
- oldcontext = MemoryContextSwitchTo(ind_context);
- partitions = lappend_oid(partitions, partoid);
- MemoryContextSwitchTo(oldcontext);
+ pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
+ npart++);
}
- /*
- * Process each partition listed in a separate transaction. Note that
- * this commits and then starts a new transaction immediately.
- * XXX: since this is done in 2*N transactions, it could just as well
- * call ReindexRelationConcurrently directly
- */
- ReindexMultipleInternal(partitions, ¶ms);
-
/*
* CIC needs to mark a partitioned index as VALID, which itself
* requires setting READY, which is unset for CIC (even though
@@ -1680,6 +1660,7 @@ reindex_invalid_child_indexes(Oid indexRelationId)
* partitions.
* See also: validatePartitionedIndex().
*/
+ CommandCounterIncrement();
index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
CommandCounterIncrement();
index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
--
2.17.0
--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13-0003-WIP-Add-SKIPVALID-flag-for-more-integration.patch"
^ permalink raw reply [nested|flat] 73+ messages in thread
* [PATCH v13 02/18] f! progress reporting
@ 2021-02-15 00:31 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 73+ messages in thread
From: Justin Pryzby @ 2021-02-15 00:31 UTC (permalink / raw)
---
src/backend/commands/indexcmds.c | 33 +++++++-------------------------
1 file changed, 7 insertions(+), 26 deletions(-)
diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 4ac1dacd7d..d6567ec231 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1638,40 +1638,20 @@ reindex_invalid_child_indexes(Oid indexRelationId)
.options = REINDEXOPT_CONCURRENTLY
};
- MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX",
- ALLOCSET_DEFAULT_SIZES);
- MemoryContext oldcontext;
- List *childs = find_inheritance_children(indexRelationId, ShareLock);
- List *partitions = NIL;
-
PreventInTransactionBlock(true, "REINDEX INDEX");
- foreach (lc, childs)
+ foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
{
Oid partoid = lfirst_oid(lc);
- /* XXX: need to retrofit progress reporting into it */
- // pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
- // npart++);
-
- if (get_index_isvalid(partoid) ||
- !RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
- continue;
+ if (!get_index_isvalid(partoid) &&
+ RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
+ ReindexRelationConcurrently(partoid, ¶ms);
- /* Save partition OID */
- oldcontext = MemoryContextSwitchTo(ind_context);
- partitions = lappend_oid(partitions, partoid);
- MemoryContextSwitchTo(oldcontext);
+ pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
+ npart++);
}
- /*
- * Process each partition listed in a separate transaction. Note that
- * this commits and then starts a new transaction immediately.
- * XXX: since this is done in 2*N transactions, it could just as well
- * call ReindexRelationConcurrently directly
- */
- ReindexMultipleInternal(partitions, ¶ms);
-
/*
* CIC needs to mark a partitioned index as VALID, which itself
* requires setting READY, which is unset for CIC (even though
@@ -1680,6 +1660,7 @@ reindex_invalid_child_indexes(Oid indexRelationId)
* partitions.
* See also: validatePartitionedIndex().
*/
+ CommandCounterIncrement();
index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
CommandCounterIncrement();
index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
--
2.17.0
--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13-0003-WIP-Add-SKIPVALID-flag-for-more-integration.patch"
^ permalink raw reply [nested|flat] 73+ messages in thread
* [PATCH v13 02/18] f! progress reporting
@ 2021-02-15 00:31 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 73+ messages in thread
From: Justin Pryzby @ 2021-02-15 00:31 UTC (permalink / raw)
---
src/backend/commands/indexcmds.c | 33 +++++++-------------------------
1 file changed, 7 insertions(+), 26 deletions(-)
diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 4ac1dacd7d..d6567ec231 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1638,40 +1638,20 @@ reindex_invalid_child_indexes(Oid indexRelationId)
.options = REINDEXOPT_CONCURRENTLY
};
- MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX",
- ALLOCSET_DEFAULT_SIZES);
- MemoryContext oldcontext;
- List *childs = find_inheritance_children(indexRelationId, ShareLock);
- List *partitions = NIL;
-
PreventInTransactionBlock(true, "REINDEX INDEX");
- foreach (lc, childs)
+ foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
{
Oid partoid = lfirst_oid(lc);
- /* XXX: need to retrofit progress reporting into it */
- // pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
- // npart++);
-
- if (get_index_isvalid(partoid) ||
- !RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
- continue;
+ if (!get_index_isvalid(partoid) &&
+ RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
+ ReindexRelationConcurrently(partoid, ¶ms);
- /* Save partition OID */
- oldcontext = MemoryContextSwitchTo(ind_context);
- partitions = lappend_oid(partitions, partoid);
- MemoryContextSwitchTo(oldcontext);
+ pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
+ npart++);
}
- /*
- * Process each partition listed in a separate transaction. Note that
- * this commits and then starts a new transaction immediately.
- * XXX: since this is done in 2*N transactions, it could just as well
- * call ReindexRelationConcurrently directly
- */
- ReindexMultipleInternal(partitions, ¶ms);
-
/*
* CIC needs to mark a partitioned index as VALID, which itself
* requires setting READY, which is unset for CIC (even though
@@ -1680,6 +1660,7 @@ reindex_invalid_child_indexes(Oid indexRelationId)
* partitions.
* See also: validatePartitionedIndex().
*/
+ CommandCounterIncrement();
index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
CommandCounterIncrement();
index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
--
2.17.0
--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13-0003-WIP-Add-SKIPVALID-flag-for-more-integration.patch"
^ permalink raw reply [nested|flat] 73+ messages in thread
* Re: Collecting statistics about contents of JSONB columns
@ 2022-01-05 20:22 Simon Riggs <[email protected]>
2022-01-06 19:48 ` Re: Collecting statistics about contents of JSONB columns Tomas Vondra <[email protected]>
0 siblings, 1 reply; 73+ messages in thread
From: Simon Riggs @ 2022-01-05 20:22 UTC (permalink / raw)
To: Tomas Vondra <[email protected]>; +Cc: PostgreSQL Hackers <[email protected]>; Nikita Glukhov <[email protected]>; Mahendra Thalor <[email protected]>; Oleg Bartunov <[email protected]>
On Fri, 31 Dec 2021 at 22:07, Tomas Vondra
<[email protected]> wrote:
> The patch does something far more
> elegant - it simply uses stavalues to store an array of JSONB documents,
> each describing stats for one path extracted from the sampled documents.
Sounds good.
> I'm sure there's plenty open questions - for example I think we'll need
> some logic to decide which paths to keep, otherwise the statistics can
> get quite big, if we're dealing with large / variable documents. We're
> already doing something similar for MCV lists.
>
> One of Nikita's patches not included in this thread allow "selective"
> statistics, where you can define in advance a "filter" restricting which
> parts are considered interesting by ANALYZE. That's interesting, but I
> think we need some simple MCV-like heuristics first anyway.
>
> Another open question is how deep the stats should be. Imagine documents
> like this:
>
> {"a" : {"b" : {"c" : {"d" : ...}}}}
>
> The current patch build stats for all possible paths:
>
> "a"
> "a.b"
> "a.b.c"
> "a.b.c.d"
>
> and of course many of the paths will often have JSONB documents as
> values, not simple scalar values. I wonder if we should limit the depth
> somehow, and maybe build stats only for scalar values.
The user interface for designing filters sounds hard, so I'd hope we
can ignore that for now.
--
Simon Riggs http://www.EnterpriseDB.com/
^ permalink raw reply [nested|flat] 73+ messages in thread
* Re: Collecting statistics about contents of JSONB columns
2022-01-05 20:22 Re: Collecting statistics about contents of JSONB columns Simon Riggs <[email protected]>
@ 2022-01-06 19:48 ` Tomas Vondra <[email protected]>
2022-01-25 20:06 ` Re: Collecting statistics about contents of JSONB columns Greg Stark <[email protected]>
0 siblings, 1 reply; 73+ messages in thread
From: Tomas Vondra @ 2022-01-06 19:48 UTC (permalink / raw)
To: Simon Riggs <[email protected]>; +Cc: PostgreSQL Hackers <[email protected]>; Nikita Glukhov <[email protected]>; Mahendra Thalor <[email protected]>; Oleg Bartunov <[email protected]>
On 1/5/22 21:22, Simon Riggs wrote:
> On Fri, 31 Dec 2021 at 22:07, Tomas Vondra
> <[email protected]> wrote:
>
>> The patch does something far more
>> elegant - it simply uses stavalues to store an array of JSONB documents,
>> each describing stats for one path extracted from the sampled documents.
>
> Sounds good.
>
>> I'm sure there's plenty open questions - for example I think we'll need
>> some logic to decide which paths to keep, otherwise the statistics can
>> get quite big, if we're dealing with large / variable documents. We're
>> already doing something similar for MCV lists.
>>
>> One of Nikita's patches not included in this thread allow "selective"
>> statistics, where you can define in advance a "filter" restricting which
>> parts are considered interesting by ANALYZE. That's interesting, but I
>> think we need some simple MCV-like heuristics first anyway.
>>
>> Another open question is how deep the stats should be. Imagine documents
>> like this:
>>
>> {"a" : {"b" : {"c" : {"d" : ...}}}}
>>
>> The current patch build stats for all possible paths:
>>
>> "a"
>> "a.b"
>> "a.b.c"
>> "a.b.c.d"
>>
>> and of course many of the paths will often have JSONB documents as
>> values, not simple scalar values. I wonder if we should limit the depth
>> somehow, and maybe build stats only for scalar values.
>
> The user interface for designing filters sounds hard, so I'd hope we
> can ignore that for now.
>
Not sure I understand. I wasn't suggesting any user-defined filtering,
but something done by default, similarly to what we do for regular MCV
lists, based on frequency. We'd include frequent paths while excluding
rare ones.
So no need for a user interface.
That might not work for documents with stable schema and a lot of
top-level paths, because all the top-level paths have 1.0 frequency. But
for documents with dynamic schema (different documents having different
schemas/paths) it might help.
Similarly for the non-scalar values - I don't think we can really keep
regular statistics on such values (for the same reason why it's not
enough for whole JSONB columns), so why to build/store that anyway.
Nikita did implement a way to specify custom filters using jsonpath, but
I did not include that into this patch series. And questions regarding
the interface were one of the reasons.
regards
--
Tomas Vondra
EnterpriseDB: http://www.enterprisedb.com
The Enterprise PostgreSQL Company
^ permalink raw reply [nested|flat] 73+ messages in thread
* Re: Collecting statistics about contents of JSONB columns
2022-01-05 20:22 Re: Collecting statistics about contents of JSONB columns Simon Riggs <[email protected]>
2022-01-06 19:48 ` Re: Collecting statistics about contents of JSONB columns Tomas Vondra <[email protected]>
@ 2022-01-25 20:06 ` Greg Stark <[email protected]>
0 siblings, 0 replies; 73+ messages in thread
From: Greg Stark @ 2022-01-25 20:06 UTC (permalink / raw)
To: Tomas Vondra <[email protected]>; +Cc: Simon Riggs <[email protected]>; PostgreSQL Hackers <[email protected]>; Nikita Glukhov <[email protected]>; Mahendra Thalor <[email protected]>; Oleg Bartunov <[email protected]>
On Thu, 6 Jan 2022 at 14:56, Tomas Vondra <[email protected]> wrote:
>
>
> Not sure I understand. I wasn't suggesting any user-defined filtering,
> but something done by default, similarly to what we do for regular MCV
> lists, based on frequency. We'd include frequent paths while excluding
> rare ones.
>
> So no need for a user interface.
Not sure but I think he was agreeing with you. That we should figure
out the baseline behaviour and get it as useful as possible first then
later look at adding some way to customize it. I agree -- I don't
think the user interface will be hard technically but I think it will
require some good ideas and there could be lots of bikeshedding. And a
lot of users will never even use it anyways so it's important to get
the defaults as useful as possible.
> Similarly for the non-scalar values - I don't think we can really keep
> regular statistics on such values (for the same reason why it's not
> enough for whole JSONB columns), so why to build/store that anyway.
For a default behaviour I wonder if it wouldn't be better to just
flatten and extract all the scalars. So if there's no matching path
then at least we have some way to estimate how often a scalar appears
anywhere in the json document.
That amounts to assuming the user knows the right path to find a given
scalar and there isn't a lot of overlap between keys. So it would at
least do something useful if you have something like {gender: female,
name: {first: nancy, last: reagan], state: california, country: usa}.
It might get things slightly wrong if you have some people named
"georgia" or have names that can be first or last names.
But it would generally be doing something more or less useful as long
as they look for "usa" in the country field and "male" in the gender
field. If they looked for "male" in $.name.first path it would give
bad estimates but assuming they know their data structure they won't
be doing that.
--
greg
^ permalink raw reply [nested|flat] 73+ messages in thread
end of thread, other threads:[~2022-01-25 20:06 UTC | newest]
Thread overview: 73+ messages (download: mbox mbox.gz follow: Atom feed)
-- links below jump to the message on this page --
2021-02-09 02:45 [PATCH v9 3/8] f! progress reporting.. Justin Pryzby <[email protected]>
2021-02-09 02:45 [PATCH v9 3/8] f! progress reporting.. Justin Pryzby <[email protected]>
2021-02-09 02:45 [PATCH v9 3/8] f! progress reporting.. Justin Pryzby <[email protected]>
2021-02-09 02:45 [PATCH v8 3/8] f! progress reporting.. Justin Pryzby <[email protected]>
2021-02-09 02:45 [PATCH v8 3/8] f! progress reporting.. Justin Pryzby <[email protected]>
2021-02-09 02:45 [PATCH v8 3/8] f! progress reporting.. Justin Pryzby <[email protected]>
2021-02-09 02:45 [PATCH v8 3/8] f! progress reporting.. Justin Pryzby <[email protected]>
2021-02-09 02:45 [PATCH v8 3/8] f! progress reporting.. Justin Pryzby <[email protected]>
2021-02-09 02:45 [PATCH v8 3/8] f! progress reporting.. Justin Pryzby <[email protected]>
2021-02-09 02:45 [PATCH v9 3/8] f! progress reporting.. Justin Pryzby <[email protected]>
2021-02-09 02:45 [PATCH v9 3/8] f! progress reporting.. Justin Pryzby <[email protected]>
2021-02-09 02:45 [PATCH v9 3/8] f! progress reporting.. Justin Pryzby <[email protected]>
2021-02-09 02:45 [PATCH v9 3/8] f! progress reporting.. Justin Pryzby <[email protected]>
2021-02-09 02:45 [PATCH v8 3/8] f! progress reporting.. Justin Pryzby <[email protected]>
2021-02-09 02:45 [PATCH v9 3/8] f! progress reporting.. Justin Pryzby <[email protected]>
2021-02-09 02:45 [PATCH v9 3/8] f! progress reporting.. Justin Pryzby <[email protected]>
2021-02-09 02:45 [PATCH v9 3/8] f! progress reporting.. Justin Pryzby <[email protected]>
2021-02-09 02:45 [PATCH v8 3/8] f! progress reporting.. Justin Pryzby <[email protected]>
2021-02-09 02:45 [PATCH v8 3/8] f! progress reporting.. Justin Pryzby <[email protected]>
2021-02-09 02:45 [PATCH v8 3/8] f! progress reporting.. Justin Pryzby <[email protected]>
2021-02-09 02:45 [PATCH v9 3/8] f! progress reporting.. Justin Pryzby <[email protected]>
2021-02-09 02:45 [PATCH v9 3/8] f! progress reporting.. Justin Pryzby <[email protected]>
2021-02-09 02:45 [PATCH v9 3/8] f! progress reporting.. Justin Pryzby <[email protected]>
2021-02-09 02:45 [PATCH v8 3/8] f! progress reporting.. Justin Pryzby <[email protected]>
2021-02-09 02:45 [PATCH v9 3/8] f! progress reporting.. Justin Pryzby <[email protected]>
2021-02-09 02:45 [PATCH v8 3/8] f! progress reporting.. Justin Pryzby <[email protected]>
2021-02-09 02:45 [PATCH v8 3/8] f! progress reporting.. Justin Pryzby <[email protected]>
2021-02-09 02:45 [PATCH v9 3/8] f! progress reporting.. Justin Pryzby <[email protected]>
2021-02-09 02:45 [PATCH v9 3/8] f! progress reporting.. Justin Pryzby <[email protected]>
2021-02-09 02:45 [PATCH v8 3/8] f! progress reporting.. Justin Pryzby <[email protected]>
2021-02-09 02:45 [PATCH v8 3/8] f! progress reporting.. Justin Pryzby <[email protected]>
2021-02-09 02:45 [PATCH v9 3/8] f! progress reporting.. Justin Pryzby <[email protected]>
2021-02-09 02:45 [PATCH v8 3/8] f! progress reporting.. Justin Pryzby <[email protected]>
2021-02-09 02:45 [PATCH v9 3/8] f! progress reporting.. Justin Pryzby <[email protected]>
2021-02-09 02:45 [PATCH v8 3/8] f! progress reporting.. Justin Pryzby <[email protected]>
2021-02-09 02:45 [PATCH v9 3/8] f! progress reporting.. Justin Pryzby <[email protected]>
2021-02-09 02:45 [PATCH v8 3/8] f! progress reporting.. Justin Pryzby <[email protected]>
2021-02-09 02:45 [PATCH v9 3/8] f! progress reporting.. Justin Pryzby <[email protected]>
2021-02-09 02:45 [PATCH v8 3/8] f! progress reporting.. Justin Pryzby <[email protected]>
2021-02-09 02:45 [PATCH v9 3/8] f! progress reporting.. Justin Pryzby <[email protected]>
2021-02-09 02:45 [PATCH v9 3/8] f! progress reporting.. Justin Pryzby <[email protected]>
2021-02-09 02:45 [PATCH v9 3/8] f! progress reporting.. Justin Pryzby <[email protected]>
2021-02-09 02:45 [PATCH v8 3/8] f! progress reporting.. Justin Pryzby <[email protected]>
2021-02-09 02:45 [PATCH v9 3/8] f! progress reporting.. Justin Pryzby <[email protected]>
2021-02-09 02:45 [PATCH v8 3/8] f! progress reporting.. Justin Pryzby <[email protected]>
2021-02-09 02:45 [PATCH v10 3/8] f! progress reporting.. Justin Pryzby <[email protected]>
2021-02-09 02:45 [PATCH v8 3/8] f! progress reporting.. Justin Pryzby <[email protected]>
2021-02-15 00:31 [PATCH v13 02/18] f! progress reporting Justin Pryzby <[email protected]>
2021-02-15 00:31 [PATCH v13 02/18] f! progress reporting Justin Pryzby <[email protected]>
2021-02-15 00:31 [PATCH v13 02/18] f! progress reporting Justin Pryzby <[email protected]>
2021-02-15 00:31 [PATCH v13 02/18] f! progress reporting Justin Pryzby <[email protected]>
2021-02-15 00:31 [PATCH v13 02/18] f! progress reporting Justin Pryzby <[email protected]>
2021-02-15 00:31 [PATCH 2/5] f! progress reporting Justin Pryzby <[email protected]>
2021-02-15 00:31 [PATCH v13 02/18] f! progress reporting Justin Pryzby <[email protected]>
2021-02-15 00:31 [PATCH v13 02/18] f! progress reporting Justin Pryzby <[email protected]>
2021-02-15 00:31 [PATCH v13 02/18] f! progress reporting Justin Pryzby <[email protected]>
2021-02-15 00:31 [PATCH v13 02/18] f! progress reporting Justin Pryzby <[email protected]>
2021-02-15 00:31 [PATCH v13 02/18] f! progress reporting Justin Pryzby <[email protected]>
2021-02-15 00:31 [PATCH v13 02/18] f! progress reporting Justin Pryzby <[email protected]>
2021-02-15 00:31 [PATCH v13 02/18] f! progress reporting Justin Pryzby <[email protected]>
2021-02-15 00:31 [PATCH v13 02/18] f! progress reporting Justin Pryzby <[email protected]>
2021-02-15 00:31 [PATCH v13 02/18] f! progress reporting Justin Pryzby <[email protected]>
2021-02-15 00:31 [PATCH v13 02/18] f! progress reporting Justin Pryzby <[email protected]>
2021-02-15 00:31 [PATCH v13 02/18] f! progress reporting Justin Pryzby <[email protected]>
2021-02-15 00:31 [PATCH v13 02/18] f! progress reporting Justin Pryzby <[email protected]>
2021-02-15 00:31 [PATCH v13 02/18] f! progress reporting Justin Pryzby <[email protected]>
2021-02-15 00:31 [PATCH v13 02/18] f! progress reporting Justin Pryzby <[email protected]>
2021-02-15 00:31 [PATCH v13 02/18] f! progress reporting Justin Pryzby <[email protected]>
2021-02-15 00:31 [PATCH v13 02/18] f! progress reporting Justin Pryzby <[email protected]>
2021-02-15 00:31 [PATCH v13 02/18] f! progress reporting Justin Pryzby <[email protected]>
2022-01-05 20:22 Re: Collecting statistics about contents of JSONB columns Simon Riggs <[email protected]>
2022-01-06 19:48 ` Re: Collecting statistics about contents of JSONB columns Tomas Vondra <[email protected]>
2022-01-25 20:06 ` Re: Collecting statistics about contents of JSONB columns Greg Stark <[email protected]>
This inbox is served by agora; see mirroring instructions
for how to clone and mirror all data and code used for this inbox