public inbox for [email protected]
help / color / mirror / Atom feedAdd index scan progress to pg_stat_progress_vacuum
44+ messages / 12 participants
[nested] [flat]
* Add index scan progress to pg_stat_progress_vacuum
@ 2021-12-01 19:32 Imseih (AWS), Sami <[email protected]>
0 siblings, 3 replies; 44+ messages in thread
From: Imseih (AWS), Sami @ 2021-12-01 19:32 UTC (permalink / raw)
To: pgsql-hackers
The current implementation of pg_stat_progress_vacuum does not provide progress on which index is being vacuumed making it difficult for a user to determine if the "vacuuming indexes" phase is making progress. By exposing which index is being scanned as well as the total progress the scan has made for the current cycle, a user can make better estimations on when the vacuum will complete.
The proposed patch adds 4 new columns to pg_stat_progress_vacuum:
1. indrelid - the relid of the index being vacuumed
2. index_blks_total - total number of blocks to be scanned in the current cycle
3. index_blks_scanned - number of blocks scanned in the current cycle
4. leader_pid - if the pid for the pg_stat_progress_vacuum entry is a leader or a vacuum worker. This patch places an entry for every worker pid ( if parallel ) as well as the leader pid
Attached is the patch.
Here is a sample output of a parallel vacuum for table with relid = 16638
postgres=# select * from pg_stat_progress_vacuum ;
-[ RECORD 1 ]------+------------------
pid | 18180
datid | 13732
datname | postgres
relid | 16638
phase | vacuuming indexes
heap_blks_total | 5149825
heap_blks_scanned | 5149825
heap_blks_vacuumed | 3686381
index_vacuum_count | 2
max_dead_tuples | 178956969
num_dead_tuples | 142086544
indrelid | 0 <<-----
index_blks_total | 0 <<-----
index_blks_scanned | 0 <<-----
leader_pid | <<-----
-[ RECORD 2 ]------+------------------
pid | 1543
datid | 13732
datname | postgres
relid | 16638
phase | vacuuming indexes
heap_blks_total | 0
heap_blks_scanned | 0
heap_blks_vacuumed | 0
index_vacuum_count | 0
max_dead_tuples | 0
num_dead_tuples | 0
indrelid | 16646
index_blks_total | 3030305
index_blks_scanned | 2356564
leader_pid | 18180
-[ RECORD 3 ]------+------------------
pid | 1544
datid | 13732
datname | postgres
relid | 16638
phase | vacuuming indexes
heap_blks_total | 0
heap_blks_scanned | 0
heap_blks_vacuumed | 0
index_vacuum_count | 0
max_dead_tuples | 0
num_dead_tuples | 0
indrelid | 16651
index_blks_total | 2685921
index_blks_scanned | 2119179
leader_pid | 18180
Regards,
Sami Imseih
Database Engineer @ Amazon Web Services
diff --git a/src/backend/access/brin/brin.c b/src/backend/access/brin/brin.c
index ccc9fa0959..a4025e376c 100644
--- a/src/backend/access/brin/brin.c
+++ b/src/backend/access/brin/brin.c
@@ -39,6 +39,8 @@
#include "utils/index_selfuncs.h"
#include "utils/memutils.h"
#include "utils/rel.h"
+#include "commands/progress.h"
+#include "pgstat.h"
/*
@@ -77,7 +79,7 @@ static void brinsummarize(Relation index, Relation heapRel, BlockNumber pageRang
static void form_and_insert_tuple(BrinBuildState *state);
static void union_tuples(BrinDesc *bdesc, BrinMemTuple *a,
BrinTuple *b);
-static void brin_vacuum_scan(Relation idxrel, BufferAccessStrategy strategy);
+static void brin_vacuum_scan(Relation idxrel, BufferAccessStrategy strategy, bool report_progress);
static bool add_values_to_range(Relation idxRel, BrinDesc *bdesc,
BrinMemTuple *dtup, Datum *values, bool *nulls);
static bool check_null_keys(BrinValues *bval, ScanKey *nullkeys, int nnullkeys);
@@ -952,7 +954,7 @@ brinvacuumcleanup(IndexVacuumInfo *info, IndexBulkDeleteResult *stats)
heapRel = table_open(IndexGetRelation(RelationGetRelid(info->index), false),
AccessShareLock);
- brin_vacuum_scan(info->index, info->strategy);
+ brin_vacuum_scan(info->index, info->strategy, info->report_progress);
brinsummarize(info->index, heapRel, BRIN_ALL_BLOCKRANGES, false,
&stats->num_index_tuples, &stats->num_index_tuples);
@@ -1634,7 +1636,7 @@ union_tuples(BrinDesc *bdesc, BrinMemTuple *a, BrinTuple *b)
* and such.
*/
static void
-brin_vacuum_scan(Relation idxrel, BufferAccessStrategy strategy)
+brin_vacuum_scan(Relation idxrel, BufferAccessStrategy strategy, bool report_progress)
{
BlockNumber nblocks;
BlockNumber blkno;
@@ -1644,6 +1646,9 @@ brin_vacuum_scan(Relation idxrel, BufferAccessStrategy strategy)
* each page.
*/
nblocks = RelationGetNumberOfBlocks(idxrel);
+ if (report_progress)
+ pgstat_progress_update_param(PROGRESS_SCAN_BLOCKS_TOTAL,
+ nblocks);
for (blkno = 0; blkno < nblocks; blkno++)
{
Buffer buf;
@@ -1655,9 +1660,21 @@ brin_vacuum_scan(Relation idxrel, BufferAccessStrategy strategy)
brin_page_cleanup(idxrel, buf);
+ if (report_progress)
+ pgstat_progress_update_param(PROGRESS_SCAN_BLOCKS_DONE,
+ blkno + 1);
+
ReleaseBuffer(buf);
}
+ if (report_progress)
+ {
+ pgstat_progress_update_param(PROGRESS_SCAN_BLOCKS_DONE,
+ 0);
+ pgstat_progress_update_param(PROGRESS_SCAN_BLOCKS_TOTAL,
+ 0);
+ }
+
/*
* Update all upper pages in the index's FSM, as well. This ensures not
* only that we propagate leaf-page FSM updates made by brin_page_cleanup,
diff --git a/src/backend/access/gin/ginvacuum.c b/src/backend/access/gin/ginvacuum.c
index a276eb020b..0171454999 100644
--- a/src/backend/access/gin/ginvacuum.c
+++ b/src/backend/access/gin/ginvacuum.c
@@ -24,6 +24,8 @@
#include "storage/lmgr.h"
#include "storage/predicate.h"
#include "utils/memutils.h"
+#include "commands/progress.h"
+#include "pgstat.h"
struct GinVacuumState
{
@@ -571,6 +573,9 @@ ginbulkdelete(IndexVacuumInfo *info, IndexBulkDeleteResult *stats,
Buffer buffer;
BlockNumber rootOfPostingTree[BLCKSZ / (sizeof(IndexTupleData) + sizeof(ItemId))];
uint32 nRoot;
+ BlockNumber num_pages;
+ bool needLock;
+ int blocks_scanned = 0;
gvs.tmpCxt = AllocSetContextCreate(CurrentMemoryContext,
"Gin vacuum temporary context",
@@ -635,6 +640,19 @@ ginbulkdelete(IndexVacuumInfo *info, IndexBulkDeleteResult *stats,
RBM_NORMAL, info->strategy);
}
+ needLock = !RELATION_IS_LOCAL(index);
+
+ /* Get the current relation length */
+ if (needLock)
+ LockRelationForExtension(index, ExclusiveLock);
+ num_pages = RelationGetNumberOfBlocks(index);
+ if (needLock)
+ UnlockRelationForExtension(index, ExclusiveLock);
+
+ if (info->report_progress)
+ pgstat_progress_update_param(PROGRESS_SCAN_BLOCKS_TOTAL,
+ num_pages);
+
/* right now we found leftmost page in entry's BTree */
for (;;)
@@ -676,9 +694,21 @@ ginbulkdelete(IndexVacuumInfo *info, IndexBulkDeleteResult *stats,
buffer = ReadBufferExtended(index, MAIN_FORKNUM, blkno,
RBM_NORMAL, info->strategy);
+ blocks_scanned++;
+ if (info->report_progress)
+ pgstat_progress_update_param(PROGRESS_SCAN_BLOCKS_DONE,
+ blocks_scanned + 1);
LockBuffer(buffer, GIN_EXCLUSIVE);
}
+ if (info->report_progress)
+ {
+ pgstat_progress_update_param(PROGRESS_SCAN_BLOCKS_DONE,
+ 0);
+ pgstat_progress_update_param(PROGRESS_SCAN_BLOCKS_TOTAL,
+ 0);
+ }
+
MemoryContextDelete(gvs.tmpCxt);
return gvs.result;
@@ -694,6 +724,7 @@ ginvacuumcleanup(IndexVacuumInfo *info, IndexBulkDeleteResult *stats)
BlockNumber totFreePages;
GinState ginstate;
GinStatsData idxStat;
+ int blocks_scanned = 0;
/*
* In an autovacuum analyze, we want to clean up pending insertions.
@@ -744,6 +775,9 @@ ginvacuumcleanup(IndexVacuumInfo *info, IndexBulkDeleteResult *stats)
totFreePages = 0;
+ if (info->report_progress)
+ pgstat_progress_update_param(PROGRESS_SCAN_BLOCKS_TOTAL,
+ npages);
for (blkno = GIN_ROOT_BLKNO; blkno < npages; blkno++)
{
Buffer buffer;
@@ -774,9 +808,22 @@ ginvacuumcleanup(IndexVacuumInfo *info, IndexBulkDeleteResult *stats)
idxStat.nEntries += PageGetMaxOffsetNumber(page);
}
+ blocks_scanned++;
+ if (info->report_progress)
+ pgstat_progress_update_param(PROGRESS_SCAN_BLOCKS_DONE,
+ blocks_scanned + 1);
+
UnlockReleaseBuffer(buffer);
}
+ if (info->report_progress)
+ {
+ pgstat_progress_update_param(PROGRESS_SCAN_BLOCKS_DONE,
+ 0);
+ pgstat_progress_update_param(PROGRESS_SCAN_BLOCKS_TOTAL,
+ 0);
+ }
+
/* Update the metapage with accurate page and entry counts */
idxStat.nTotalPages = npages;
ginUpdateStats(info->index, &idxStat, false);
diff --git a/src/backend/access/gist/gistvacuum.c b/src/backend/access/gist/gistvacuum.c
index 0663193531..3be8bb4282 100644
--- a/src/backend/access/gist/gistvacuum.c
+++ b/src/backend/access/gist/gistvacuum.c
@@ -23,6 +23,8 @@
#include "storage/indexfsm.h"
#include "storage/lmgr.h"
#include "utils/memutils.h"
+#include "commands/progress.h"
+#include "pgstat.h"
/* Working state needed by gistbulkdelete */
typedef struct
@@ -215,9 +217,27 @@ gistvacuumscan(IndexVacuumInfo *info, IndexBulkDeleteResult *stats,
/* Quit if we've scanned the whole relation */
if (blkno >= num_pages)
break;
+
+ if (info->report_progress)
+ pgstat_progress_update_param(PROGRESS_SCAN_BLOCKS_TOTAL,
+ num_pages);
+
/* Iterate over pages, then loop back to recheck length */
for (; blkno < num_pages; blkno++)
+ {
gistvacuumpage(&vstate, blkno, blkno);
+ if (info->report_progress)
+ pgstat_progress_update_param(PROGRESS_SCAN_BLOCKS_DONE,
+ blkno + 1);
+ }
+ }
+
+ if (info->report_progress)
+ {
+ pgstat_progress_update_param(PROGRESS_SCAN_BLOCKS_DONE,
+ 0);
+ pgstat_progress_update_param(PROGRESS_SCAN_BLOCKS_TOTAL,
+ 0);
}
/*
diff --git a/src/backend/access/hash/hash.c b/src/backend/access/hash/hash.c
index eb3810494f..851f33fd97 100644
--- a/src/backend/access/hash/hash.c
+++ b/src/backend/access/hash/hash.c
@@ -31,6 +31,7 @@
#include "utils/builtins.h"
#include "utils/index_selfuncs.h"
#include "utils/rel.h"
+#include "storage/lmgr.h"
/* Working state for hashbuild and its callback */
typedef struct
@@ -468,9 +469,15 @@ hashbulkdelete(IndexVacuumInfo *info, IndexBulkDeleteResult *stats,
Buffer metabuf = InvalidBuffer;
HashMetaPage metap;
HashMetaPage cachedmetap;
+ int blocks_scanned;
+ int bucket_blocks_scanned;
+ BlockNumber num_pages;
+ bool needLock;
tuples_removed = 0;
num_index_tuples = 0;
+ blocks_scanned = 0;
+ bucket_blocks_scanned = 0;
/*
* We need a copy of the metapage so that we can use its hashm_spares[]
@@ -488,6 +495,19 @@ hashbulkdelete(IndexVacuumInfo *info, IndexBulkDeleteResult *stats,
cur_bucket = 0;
cur_maxbucket = orig_maxbucket;
+ needLock = !RELATION_IS_LOCAL(rel);
+
+ /* Get the current relation length */
+ if (needLock)
+ LockRelationForExtension(rel, ExclusiveLock);
+ num_pages = RelationGetNumberOfBlocks(rel);
+ if (needLock)
+ UnlockRelationForExtension(rel, ExclusiveLock);
+
+ if (info->report_progress)
+ pgstat_progress_update_param(PROGRESS_SCAN_BLOCKS_TOTAL,
+ num_pages);
+
loop_top:
while (cur_bucket <= cur_maxbucket)
{
@@ -503,6 +523,7 @@ loop_top:
bucket_blkno = BUCKET_TO_BLKNO(cachedmetap, cur_bucket);
blkno = bucket_blkno;
+ blocks_scanned++;
/*
* We need to acquire a cleanup lock on the primary bucket page to out
@@ -549,10 +570,14 @@ loop_top:
cachedmetap->hashm_highmask,
cachedmetap->hashm_lowmask, &tuples_removed,
&num_index_tuples, split_cleanup,
- callback, callback_state);
+ callback, callback_state, info->report_progress,
+ &bucket_blocks_scanned);
_hash_dropbuf(rel, bucket_buf);
+ pgstat_progress_update_param(PROGRESS_SCAN_BLOCKS_DONE,
+ blocks_scanned + bucket_blocks_scanned);
+
/* Advance to next bucket */
cur_bucket++;
}
@@ -632,6 +657,14 @@ loop_top:
stats->tuples_removed += tuples_removed;
/* hashvacuumcleanup will fill in num_pages */
+ if (info->report_progress)
+ {
+ pgstat_progress_update_param(PROGRESS_SCAN_BLOCKS_DONE,
+ 0);
+ pgstat_progress_update_param(PROGRESS_SCAN_BLOCKS_TOTAL,
+ 0);
+ }
+
return stats;
}
@@ -685,7 +718,8 @@ hashbucketcleanup(Relation rel, Bucket cur_bucket, Buffer bucket_buf,
uint32 maxbucket, uint32 highmask, uint32 lowmask,
double *tuples_removed, double *num_index_tuples,
bool split_cleanup,
- IndexBulkDeleteCallback callback, void *callback_state)
+ IndexBulkDeleteCallback callback, void *callback_state,
+ bool report_progress, int *bucket_blocks_scanned)
{
BlockNumber blkno;
Buffer buf;
@@ -717,6 +751,8 @@ hashbucketcleanup(Relation rel, Bucket cur_bucket, Buffer bucket_buf,
page = BufferGetPage(buf);
opaque = (HashPageOpaque) PageGetSpecialPointer(page);
+ bucket_blocks_scanned++;
+
/* Scan each tuple in page */
maxoffno = PageGetMaxOffsetNumber(page);
for (offno = FirstOffsetNumber;
@@ -915,4 +951,5 @@ hashbucketcleanup(Relation rel, Bucket cur_bucket, Buffer bucket_buf,
bstrategy);
else
LockBuffer(bucket_buf, BUFFER_LOCK_UNLOCK);
+
}
diff --git a/src/backend/access/hash/hashpage.c b/src/backend/access/hash/hashpage.c
index 159646c7c3..538d153df7 100644
--- a/src/backend/access/hash/hashpage.c
+++ b/src/backend/access/hash/hashpage.c
@@ -758,7 +758,7 @@ restart_expand:
hashbucketcleanup(rel, old_bucket, buf_oblkno, start_oblkno, NULL,
maxbucket, highmask, lowmask, NULL, NULL, true,
- NULL, NULL);
+ NULL, NULL, NULL, NULL);
_hash_dropbuf(rel, buf_oblkno);
@@ -1326,7 +1326,7 @@ _hash_splitbucket(Relation rel,
hashbucketcleanup(rel, obucket, bucket_obuf,
BufferGetBlockNumber(bucket_obuf), NULL,
maxbucket, highmask, lowmask, NULL, NULL, true,
- NULL, NULL);
+ NULL, NULL, NULL, NULL);
}
else
{
diff --git a/src/backend/access/heap/vacuumlazy.c b/src/backend/access/heap/vacuumlazy.c
index a00947ea1c..28a3797088 100644
--- a/src/backend/access/heap/vacuumlazy.c
+++ b/src/backend/access/heap/vacuumlazy.c
@@ -528,6 +528,9 @@ heap_vacuum_rel(Relation rel, VacuumParams *params,
pgstat_progress_start_command(PROGRESS_COMMAND_VACUUM,
RelationGetRelid(rel));
+ pgstat_progress_update_param(PROGRESS_VACUUM_LEADER_PID,
+ MyProcPid);
+
vacuum_set_xid_limits(rel,
params->freeze_min_age,
params->freeze_table_age,
@@ -3025,12 +3028,17 @@ lazy_vacuum_one_index(Relation indrel, IndexBulkDeleteResult *istat,
IndexVacuumInfo ivinfo;
PGRUsage ru0;
LVSavedErrInfo saved_err_info;
+ const int initprog_index[] = {
+ PROGRESS_VACUUM_PHASE,
+ PROGRESS_VACUUM_CURRENT_INDRELID
+ };
+ int64 initprog_val[2];
pg_rusage_init(&ru0);
ivinfo.index = indrel;
ivinfo.analyze_only = false;
- ivinfo.report_progress = false;
+ ivinfo.report_progress = true;
ivinfo.estimated_count = true;
ivinfo.message_level = elevel;
ivinfo.num_heap_tuples = reltuples;
@@ -3048,10 +3056,18 @@ lazy_vacuum_one_index(Relation indrel, IndexBulkDeleteResult *istat,
VACUUM_ERRCB_PHASE_VACUUM_INDEX,
InvalidBlockNumber, InvalidOffsetNumber);
+ /* Report that we're vacuuming the index, advertising the indrelid */
+ initprog_val[0] = PROGRESS_VACUUM_PHASE_VACUUM_INDEX;
+ initprog_val[1] = RelationGetRelid(indrel);
+ pgstat_progress_update_multi_param(2, initprog_index, initprog_val);
+
/* Do bulk deletion */
istat = index_bulk_delete(&ivinfo, istat, lazy_tid_reaped,
(void *) vacrel->dead_tuples);
+ /* Report that we're done vacuuming the index */
+ pgstat_progress_update_param(PROGRESS_VACUUM_CURRENT_INDRELID,
+ 0);
ereport(elevel,
(errmsg("scanned index \"%s\" to remove %d row versions",
vacrel->indname, vacrel->dead_tuples->num_tuples),
@@ -3081,12 +3097,17 @@ lazy_cleanup_one_index(Relation indrel, IndexBulkDeleteResult *istat,
IndexVacuumInfo ivinfo;
PGRUsage ru0;
LVSavedErrInfo saved_err_info;
+ const int initprog_index[] = {
+ PROGRESS_VACUUM_PHASE,
+ PROGRESS_VACUUM_CURRENT_INDRELID
+ };
+ int64 initprog_val[2];
pg_rusage_init(&ru0);
ivinfo.index = indrel;
ivinfo.analyze_only = false;
- ivinfo.report_progress = false;
+ ivinfo.report_progress = true;
ivinfo.estimated_count = estimated_count;
ivinfo.message_level = elevel;
@@ -3105,8 +3126,18 @@ lazy_cleanup_one_index(Relation indrel, IndexBulkDeleteResult *istat,
VACUUM_ERRCB_PHASE_INDEX_CLEANUP,
InvalidBlockNumber, InvalidOffsetNumber);
+ /* Report that we're cleaning the index, advertising the indrelid */
+ initprog_val[0] = PROGRESS_VACUUM_PHASE_INDEX_CLEANUP;
+ initprog_val[1] = RelationGetRelid(indrel);
+ pgstat_progress_update_multi_param(2, initprog_index, initprog_val);
+
istat = index_vacuum_cleanup(&ivinfo, istat);
+ /* Report that we're done cleaning the index */
+ initprog_val[0] = 0;
+ initprog_val[1] = 0;
+ pgstat_progress_update_multi_param(2, initprog_index, initprog_val);
+
if (istat)
{
ereport(elevel,
@@ -4158,6 +4189,7 @@ parallel_vacuum_main(dsm_segment *seg, shm_toc *toc)
char *sharedquery;
LVRelState vacrel;
ErrorContextCallback errcallback;
+ PGPROC *leader = MyProc->lockGroupLeader;
/*
* A parallel vacuum worker must have only PROC_IN_VACUUM flag since we
@@ -4186,6 +4218,14 @@ parallel_vacuum_main(dsm_segment *seg, shm_toc *toc)
*/
rel = table_open(lvshared->relid, ShareUpdateExclusiveLock);
+ /*
+ * Track progress of current index being vacuumed
+ */
+ pgstat_progress_start_command(PROGRESS_COMMAND_VACUUM,
+ RelationGetRelid(rel));
+
+ pgstat_progress_update_param(PROGRESS_VACUUM_LEADER_PID, leader->pid);
+
/*
* Open all indexes. indrels are sorted in order by OID, which should be
* matched to the leader's one.
@@ -4252,6 +4292,7 @@ parallel_vacuum_main(dsm_segment *seg, shm_toc *toc)
vac_close_indexes(nindexes, indrels, RowExclusiveLock);
table_close(rel, ShareUpdateExclusiveLock);
FreeAccessStrategy(vacrel.bstrategy);
+ pgstat_progress_end_command();
pfree(vacrel.indstats);
}
diff --git a/src/backend/access/nbtree/nbtree.c b/src/backend/access/nbtree/nbtree.c
index 40ad0956e0..e90fd77b86 100644
--- a/src/backend/access/nbtree/nbtree.c
+++ b/src/backend/access/nbtree/nbtree.c
@@ -996,10 +996,18 @@ btvacuumscan(IndexVacuumInfo *info, IndexBulkDeleteResult *stats,
btvacuumpage(&vstate, scanblkno);
if (info->report_progress)
pgstat_progress_update_param(PROGRESS_SCAN_BLOCKS_DONE,
- scanblkno);
+ scanblkno + 1);
}
}
+ if (info->report_progress)
+ {
+ pgstat_progress_update_param(PROGRESS_SCAN_BLOCKS_DONE,
+ 0);
+ pgstat_progress_update_param(PROGRESS_SCAN_BLOCKS_TOTAL,
+ 0);
+ }
+
/* Set statistics num_pages field to final size of index */
stats->num_pages = num_pages;
diff --git a/src/backend/access/spgist/spgvacuum.c b/src/backend/access/spgist/spgvacuum.c
index 76fb0374c4..19768f11ce 100644
--- a/src/backend/access/spgist/spgvacuum.c
+++ b/src/backend/access/spgist/spgvacuum.c
@@ -27,6 +27,8 @@
#include "storage/indexfsm.h"
#include "storage/lmgr.h"
#include "utils/snapmgr.h"
+#include "commands/progress.h"
+#include "pgstat.h"
/* Entry in pending-list of TIDs we need to revisit */
@@ -797,6 +799,7 @@ spgvacuumscan(spgBulkDeleteState *bds)
bool needLock;
BlockNumber num_pages,
blkno;
+ int blocks_scanned = 0;
/* Finish setting up spgBulkDeleteState */
initSpGistState(&bds->spgstate, index);
@@ -836,6 +839,11 @@ spgvacuumscan(spgBulkDeleteState *bds)
/* Quit if we've scanned the whole relation */
if (blkno >= num_pages)
break;
+
+ if (bds->info->report_progress)
+ pgstat_progress_update_param(PROGRESS_SCAN_BLOCKS_TOTAL,
+ num_pages);
+
/* Iterate over pages, then loop back to recheck length */
for (; blkno < num_pages; blkno++)
{
@@ -843,9 +851,22 @@ spgvacuumscan(spgBulkDeleteState *bds)
/* empty the pending-list after each page */
if (bds->pendingList != NULL)
spgprocesspending(bds);
+
+ blocks_scanned++;
+ if (bds->info->report_progress)
+ pgstat_progress_update_param(PROGRESS_SCAN_BLOCKS_DONE,
+ blocks_scanned + 1);
}
}
+ if (bds->info->report_progress)
+ {
+ pgstat_progress_update_param(PROGRESS_SCAN_BLOCKS_DONE,
+ 0);
+ pgstat_progress_update_param(PROGRESS_SCAN_BLOCKS_TOTAL,
+ 0);
+ }
+
/* Propagate local lastUsedPages cache to metablock */
SpGistUpdateMetaPage(index);
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index eb560955cd..953d238925 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1124,7 +1124,10 @@ CREATE VIEW pg_stat_progress_vacuum AS
END AS phase,
S.param2 AS heap_blks_total, S.param3 AS heap_blks_scanned,
S.param4 AS heap_blks_vacuumed, S.param5 AS index_vacuum_count,
- S.param6 AS max_dead_tuples, S.param7 AS num_dead_tuples
+ S.param6 AS max_dead_tuples, S.param7 AS num_dead_tuples,
+ S.param8 AS indrelid, S.param16 index_blks_total,
+ S.param17 AS index_blks_scanned,
+ CASE WHEN S.param9 = S.pid THEN NULL ELSE S.param9 END AS leader_pid
FROM pg_stat_get_progress_info('VACUUM') AS S
LEFT JOIN pg_database D ON S.datid = D.oid;
diff --git a/src/include/access/hash.h b/src/include/access/hash.h
index 1cce865be2..c00bb76e3e 100644
--- a/src/include/access/hash.h
+++ b/src/include/access/hash.h
@@ -478,6 +478,7 @@ extern void hashbucketcleanup(Relation rel, Bucket cur_bucket,
uint32 maxbucket, uint32 highmask, uint32 lowmask,
double *tuples_removed, double *num_index_tuples,
bool split_cleanup,
- IndexBulkDeleteCallback callback, void *callback_state);
+ IndexBulkDeleteCallback callback, void *callback_state,
+ bool report_progress, int *bucket_blocks_scanned);
#endif /* HASH_H */
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index d7bf16368b..4387a7c1f1 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -25,6 +25,8 @@
#define PROGRESS_VACUUM_NUM_INDEX_VACUUMS 4
#define PROGRESS_VACUUM_MAX_DEAD_TUPLES 5
#define PROGRESS_VACUUM_NUM_DEAD_TUPLES 6
+#define PROGRESS_VACUUM_CURRENT_INDRELID 7
+#define PROGRESS_VACUUM_LEADER_PID 8
/* Phases of vacuum (as advertised via PROGRESS_VACUUM_PHASE) */
#define PROGRESS_VACUUM_PHASE_SCAN_HEAP 1
Attachments:
[text/plain] patch.txt (19.5K, ../../[email protected]/3-patch.txt)
download | inline diff:
diff --git a/src/backend/access/brin/brin.c b/src/backend/access/brin/brin.c
index ccc9fa0959..a4025e376c 100644
--- a/src/backend/access/brin/brin.c
+++ b/src/backend/access/brin/brin.c
@@ -39,6 +39,8 @@
#include "utils/index_selfuncs.h"
#include "utils/memutils.h"
#include "utils/rel.h"
+#include "commands/progress.h"
+#include "pgstat.h"
/*
@@ -77,7 +79,7 @@ static void brinsummarize(Relation index, Relation heapRel, BlockNumber pageRang
static void form_and_insert_tuple(BrinBuildState *state);
static void union_tuples(BrinDesc *bdesc, BrinMemTuple *a,
BrinTuple *b);
-static void brin_vacuum_scan(Relation idxrel, BufferAccessStrategy strategy);
+static void brin_vacuum_scan(Relation idxrel, BufferAccessStrategy strategy, bool report_progress);
static bool add_values_to_range(Relation idxRel, BrinDesc *bdesc,
BrinMemTuple *dtup, Datum *values, bool *nulls);
static bool check_null_keys(BrinValues *bval, ScanKey *nullkeys, int nnullkeys);
@@ -952,7 +954,7 @@ brinvacuumcleanup(IndexVacuumInfo *info, IndexBulkDeleteResult *stats)
heapRel = table_open(IndexGetRelation(RelationGetRelid(info->index), false),
AccessShareLock);
- brin_vacuum_scan(info->index, info->strategy);
+ brin_vacuum_scan(info->index, info->strategy, info->report_progress);
brinsummarize(info->index, heapRel, BRIN_ALL_BLOCKRANGES, false,
&stats->num_index_tuples, &stats->num_index_tuples);
@@ -1634,7 +1636,7 @@ union_tuples(BrinDesc *bdesc, BrinMemTuple *a, BrinTuple *b)
* and such.
*/
static void
-brin_vacuum_scan(Relation idxrel, BufferAccessStrategy strategy)
+brin_vacuum_scan(Relation idxrel, BufferAccessStrategy strategy, bool report_progress)
{
BlockNumber nblocks;
BlockNumber blkno;
@@ -1644,6 +1646,9 @@ brin_vacuum_scan(Relation idxrel, BufferAccessStrategy strategy)
* each page.
*/
nblocks = RelationGetNumberOfBlocks(idxrel);
+ if (report_progress)
+ pgstat_progress_update_param(PROGRESS_SCAN_BLOCKS_TOTAL,
+ nblocks);
for (blkno = 0; blkno < nblocks; blkno++)
{
Buffer buf;
@@ -1655,9 +1660,21 @@ brin_vacuum_scan(Relation idxrel, BufferAccessStrategy strategy)
brin_page_cleanup(idxrel, buf);
+ if (report_progress)
+ pgstat_progress_update_param(PROGRESS_SCAN_BLOCKS_DONE,
+ blkno + 1);
+
ReleaseBuffer(buf);
}
+ if (report_progress)
+ {
+ pgstat_progress_update_param(PROGRESS_SCAN_BLOCKS_DONE,
+ 0);
+ pgstat_progress_update_param(PROGRESS_SCAN_BLOCKS_TOTAL,
+ 0);
+ }
+
/*
* Update all upper pages in the index's FSM, as well. This ensures not
* only that we propagate leaf-page FSM updates made by brin_page_cleanup,
diff --git a/src/backend/access/gin/ginvacuum.c b/src/backend/access/gin/ginvacuum.c
index a276eb020b..0171454999 100644
--- a/src/backend/access/gin/ginvacuum.c
+++ b/src/backend/access/gin/ginvacuum.c
@@ -24,6 +24,8 @@
#include "storage/lmgr.h"
#include "storage/predicate.h"
#include "utils/memutils.h"
+#include "commands/progress.h"
+#include "pgstat.h"
struct GinVacuumState
{
@@ -571,6 +573,9 @@ ginbulkdelete(IndexVacuumInfo *info, IndexBulkDeleteResult *stats,
Buffer buffer;
BlockNumber rootOfPostingTree[BLCKSZ / (sizeof(IndexTupleData) + sizeof(ItemId))];
uint32 nRoot;
+ BlockNumber num_pages;
+ bool needLock;
+ int blocks_scanned = 0;
gvs.tmpCxt = AllocSetContextCreate(CurrentMemoryContext,
"Gin vacuum temporary context",
@@ -635,6 +640,19 @@ ginbulkdelete(IndexVacuumInfo *info, IndexBulkDeleteResult *stats,
RBM_NORMAL, info->strategy);
}
+ needLock = !RELATION_IS_LOCAL(index);
+
+ /* Get the current relation length */
+ if (needLock)
+ LockRelationForExtension(index, ExclusiveLock);
+ num_pages = RelationGetNumberOfBlocks(index);
+ if (needLock)
+ UnlockRelationForExtension(index, ExclusiveLock);
+
+ if (info->report_progress)
+ pgstat_progress_update_param(PROGRESS_SCAN_BLOCKS_TOTAL,
+ num_pages);
+
/* right now we found leftmost page in entry's BTree */
for (;;)
@@ -676,9 +694,21 @@ ginbulkdelete(IndexVacuumInfo *info, IndexBulkDeleteResult *stats,
buffer = ReadBufferExtended(index, MAIN_FORKNUM, blkno,
RBM_NORMAL, info->strategy);
+ blocks_scanned++;
+ if (info->report_progress)
+ pgstat_progress_update_param(PROGRESS_SCAN_BLOCKS_DONE,
+ blocks_scanned + 1);
LockBuffer(buffer, GIN_EXCLUSIVE);
}
+ if (info->report_progress)
+ {
+ pgstat_progress_update_param(PROGRESS_SCAN_BLOCKS_DONE,
+ 0);
+ pgstat_progress_update_param(PROGRESS_SCAN_BLOCKS_TOTAL,
+ 0);
+ }
+
MemoryContextDelete(gvs.tmpCxt);
return gvs.result;
@@ -694,6 +724,7 @@ ginvacuumcleanup(IndexVacuumInfo *info, IndexBulkDeleteResult *stats)
BlockNumber totFreePages;
GinState ginstate;
GinStatsData idxStat;
+ int blocks_scanned = 0;
/*
* In an autovacuum analyze, we want to clean up pending insertions.
@@ -744,6 +775,9 @@ ginvacuumcleanup(IndexVacuumInfo *info, IndexBulkDeleteResult *stats)
totFreePages = 0;
+ if (info->report_progress)
+ pgstat_progress_update_param(PROGRESS_SCAN_BLOCKS_TOTAL,
+ npages);
for (blkno = GIN_ROOT_BLKNO; blkno < npages; blkno++)
{
Buffer buffer;
@@ -774,9 +808,22 @@ ginvacuumcleanup(IndexVacuumInfo *info, IndexBulkDeleteResult *stats)
idxStat.nEntries += PageGetMaxOffsetNumber(page);
}
+ blocks_scanned++;
+ if (info->report_progress)
+ pgstat_progress_update_param(PROGRESS_SCAN_BLOCKS_DONE,
+ blocks_scanned + 1);
+
UnlockReleaseBuffer(buffer);
}
+ if (info->report_progress)
+ {
+ pgstat_progress_update_param(PROGRESS_SCAN_BLOCKS_DONE,
+ 0);
+ pgstat_progress_update_param(PROGRESS_SCAN_BLOCKS_TOTAL,
+ 0);
+ }
+
/* Update the metapage with accurate page and entry counts */
idxStat.nTotalPages = npages;
ginUpdateStats(info->index, &idxStat, false);
diff --git a/src/backend/access/gist/gistvacuum.c b/src/backend/access/gist/gistvacuum.c
index 0663193531..3be8bb4282 100644
--- a/src/backend/access/gist/gistvacuum.c
+++ b/src/backend/access/gist/gistvacuum.c
@@ -23,6 +23,8 @@
#include "storage/indexfsm.h"
#include "storage/lmgr.h"
#include "utils/memutils.h"
+#include "commands/progress.h"
+#include "pgstat.h"
/* Working state needed by gistbulkdelete */
typedef struct
@@ -215,9 +217,27 @@ gistvacuumscan(IndexVacuumInfo *info, IndexBulkDeleteResult *stats,
/* Quit if we've scanned the whole relation */
if (blkno >= num_pages)
break;
+
+ if (info->report_progress)
+ pgstat_progress_update_param(PROGRESS_SCAN_BLOCKS_TOTAL,
+ num_pages);
+
/* Iterate over pages, then loop back to recheck length */
for (; blkno < num_pages; blkno++)
+ {
gistvacuumpage(&vstate, blkno, blkno);
+ if (info->report_progress)
+ pgstat_progress_update_param(PROGRESS_SCAN_BLOCKS_DONE,
+ blkno + 1);
+ }
+ }
+
+ if (info->report_progress)
+ {
+ pgstat_progress_update_param(PROGRESS_SCAN_BLOCKS_DONE,
+ 0);
+ pgstat_progress_update_param(PROGRESS_SCAN_BLOCKS_TOTAL,
+ 0);
}
/*
diff --git a/src/backend/access/hash/hash.c b/src/backend/access/hash/hash.c
index eb3810494f..851f33fd97 100644
--- a/src/backend/access/hash/hash.c
+++ b/src/backend/access/hash/hash.c
@@ -31,6 +31,7 @@
#include "utils/builtins.h"
#include "utils/index_selfuncs.h"
#include "utils/rel.h"
+#include "storage/lmgr.h"
/* Working state for hashbuild and its callback */
typedef struct
@@ -468,9 +469,15 @@ hashbulkdelete(IndexVacuumInfo *info, IndexBulkDeleteResult *stats,
Buffer metabuf = InvalidBuffer;
HashMetaPage metap;
HashMetaPage cachedmetap;
+ int blocks_scanned;
+ int bucket_blocks_scanned;
+ BlockNumber num_pages;
+ bool needLock;
tuples_removed = 0;
num_index_tuples = 0;
+ blocks_scanned = 0;
+ bucket_blocks_scanned = 0;
/*
* We need a copy of the metapage so that we can use its hashm_spares[]
@@ -488,6 +495,19 @@ hashbulkdelete(IndexVacuumInfo *info, IndexBulkDeleteResult *stats,
cur_bucket = 0;
cur_maxbucket = orig_maxbucket;
+ needLock = !RELATION_IS_LOCAL(rel);
+
+ /* Get the current relation length */
+ if (needLock)
+ LockRelationForExtension(rel, ExclusiveLock);
+ num_pages = RelationGetNumberOfBlocks(rel);
+ if (needLock)
+ UnlockRelationForExtension(rel, ExclusiveLock);
+
+ if (info->report_progress)
+ pgstat_progress_update_param(PROGRESS_SCAN_BLOCKS_TOTAL,
+ num_pages);
+
loop_top:
while (cur_bucket <= cur_maxbucket)
{
@@ -503,6 +523,7 @@ loop_top:
bucket_blkno = BUCKET_TO_BLKNO(cachedmetap, cur_bucket);
blkno = bucket_blkno;
+ blocks_scanned++;
/*
* We need to acquire a cleanup lock on the primary bucket page to out
@@ -549,10 +570,14 @@ loop_top:
cachedmetap->hashm_highmask,
cachedmetap->hashm_lowmask, &tuples_removed,
&num_index_tuples, split_cleanup,
- callback, callback_state);
+ callback, callback_state, info->report_progress,
+ &bucket_blocks_scanned);
_hash_dropbuf(rel, bucket_buf);
+ pgstat_progress_update_param(PROGRESS_SCAN_BLOCKS_DONE,
+ blocks_scanned + bucket_blocks_scanned);
+
/* Advance to next bucket */
cur_bucket++;
}
@@ -632,6 +657,14 @@ loop_top:
stats->tuples_removed += tuples_removed;
/* hashvacuumcleanup will fill in num_pages */
+ if (info->report_progress)
+ {
+ pgstat_progress_update_param(PROGRESS_SCAN_BLOCKS_DONE,
+ 0);
+ pgstat_progress_update_param(PROGRESS_SCAN_BLOCKS_TOTAL,
+ 0);
+ }
+
return stats;
}
@@ -685,7 +718,8 @@ hashbucketcleanup(Relation rel, Bucket cur_bucket, Buffer bucket_buf,
uint32 maxbucket, uint32 highmask, uint32 lowmask,
double *tuples_removed, double *num_index_tuples,
bool split_cleanup,
- IndexBulkDeleteCallback callback, void *callback_state)
+ IndexBulkDeleteCallback callback, void *callback_state,
+ bool report_progress, int *bucket_blocks_scanned)
{
BlockNumber blkno;
Buffer buf;
@@ -717,6 +751,8 @@ hashbucketcleanup(Relation rel, Bucket cur_bucket, Buffer bucket_buf,
page = BufferGetPage(buf);
opaque = (HashPageOpaque) PageGetSpecialPointer(page);
+ bucket_blocks_scanned++;
+
/* Scan each tuple in page */
maxoffno = PageGetMaxOffsetNumber(page);
for (offno = FirstOffsetNumber;
@@ -915,4 +951,5 @@ hashbucketcleanup(Relation rel, Bucket cur_bucket, Buffer bucket_buf,
bstrategy);
else
LockBuffer(bucket_buf, BUFFER_LOCK_UNLOCK);
+
}
diff --git a/src/backend/access/hash/hashpage.c b/src/backend/access/hash/hashpage.c
index 159646c7c3..538d153df7 100644
--- a/src/backend/access/hash/hashpage.c
+++ b/src/backend/access/hash/hashpage.c
@@ -758,7 +758,7 @@ restart_expand:
hashbucketcleanup(rel, old_bucket, buf_oblkno, start_oblkno, NULL,
maxbucket, highmask, lowmask, NULL, NULL, true,
- NULL, NULL);
+ NULL, NULL, NULL, NULL);
_hash_dropbuf(rel, buf_oblkno);
@@ -1326,7 +1326,7 @@ _hash_splitbucket(Relation rel,
hashbucketcleanup(rel, obucket, bucket_obuf,
BufferGetBlockNumber(bucket_obuf), NULL,
maxbucket, highmask, lowmask, NULL, NULL, true,
- NULL, NULL);
+ NULL, NULL, NULL, NULL);
}
else
{
diff --git a/src/backend/access/heap/vacuumlazy.c b/src/backend/access/heap/vacuumlazy.c
index a00947ea1c..28a3797088 100644
--- a/src/backend/access/heap/vacuumlazy.c
+++ b/src/backend/access/heap/vacuumlazy.c
@@ -528,6 +528,9 @@ heap_vacuum_rel(Relation rel, VacuumParams *params,
pgstat_progress_start_command(PROGRESS_COMMAND_VACUUM,
RelationGetRelid(rel));
+ pgstat_progress_update_param(PROGRESS_VACUUM_LEADER_PID,
+ MyProcPid);
+
vacuum_set_xid_limits(rel,
params->freeze_min_age,
params->freeze_table_age,
@@ -3025,12 +3028,17 @@ lazy_vacuum_one_index(Relation indrel, IndexBulkDeleteResult *istat,
IndexVacuumInfo ivinfo;
PGRUsage ru0;
LVSavedErrInfo saved_err_info;
+ const int initprog_index[] = {
+ PROGRESS_VACUUM_PHASE,
+ PROGRESS_VACUUM_CURRENT_INDRELID
+ };
+ int64 initprog_val[2];
pg_rusage_init(&ru0);
ivinfo.index = indrel;
ivinfo.analyze_only = false;
- ivinfo.report_progress = false;
+ ivinfo.report_progress = true;
ivinfo.estimated_count = true;
ivinfo.message_level = elevel;
ivinfo.num_heap_tuples = reltuples;
@@ -3048,10 +3056,18 @@ lazy_vacuum_one_index(Relation indrel, IndexBulkDeleteResult *istat,
VACUUM_ERRCB_PHASE_VACUUM_INDEX,
InvalidBlockNumber, InvalidOffsetNumber);
+ /* Report that we're vacuuming the index, advertising the indrelid */
+ initprog_val[0] = PROGRESS_VACUUM_PHASE_VACUUM_INDEX;
+ initprog_val[1] = RelationGetRelid(indrel);
+ pgstat_progress_update_multi_param(2, initprog_index, initprog_val);
+
/* Do bulk deletion */
istat = index_bulk_delete(&ivinfo, istat, lazy_tid_reaped,
(void *) vacrel->dead_tuples);
+ /* Report that we're done vacuuming the index */
+ pgstat_progress_update_param(PROGRESS_VACUUM_CURRENT_INDRELID,
+ 0);
ereport(elevel,
(errmsg("scanned index \"%s\" to remove %d row versions",
vacrel->indname, vacrel->dead_tuples->num_tuples),
@@ -3081,12 +3097,17 @@ lazy_cleanup_one_index(Relation indrel, IndexBulkDeleteResult *istat,
IndexVacuumInfo ivinfo;
PGRUsage ru0;
LVSavedErrInfo saved_err_info;
+ const int initprog_index[] = {
+ PROGRESS_VACUUM_PHASE,
+ PROGRESS_VACUUM_CURRENT_INDRELID
+ };
+ int64 initprog_val[2];
pg_rusage_init(&ru0);
ivinfo.index = indrel;
ivinfo.analyze_only = false;
- ivinfo.report_progress = false;
+ ivinfo.report_progress = true;
ivinfo.estimated_count = estimated_count;
ivinfo.message_level = elevel;
@@ -3105,8 +3126,18 @@ lazy_cleanup_one_index(Relation indrel, IndexBulkDeleteResult *istat,
VACUUM_ERRCB_PHASE_INDEX_CLEANUP,
InvalidBlockNumber, InvalidOffsetNumber);
+ /* Report that we're cleaning the index, advertising the indrelid */
+ initprog_val[0] = PROGRESS_VACUUM_PHASE_INDEX_CLEANUP;
+ initprog_val[1] = RelationGetRelid(indrel);
+ pgstat_progress_update_multi_param(2, initprog_index, initprog_val);
+
istat = index_vacuum_cleanup(&ivinfo, istat);
+ /* Report that we're done cleaning the index */
+ initprog_val[0] = 0;
+ initprog_val[1] = 0;
+ pgstat_progress_update_multi_param(2, initprog_index, initprog_val);
+
if (istat)
{
ereport(elevel,
@@ -4158,6 +4189,7 @@ parallel_vacuum_main(dsm_segment *seg, shm_toc *toc)
char *sharedquery;
LVRelState vacrel;
ErrorContextCallback errcallback;
+ PGPROC *leader = MyProc->lockGroupLeader;
/*
* A parallel vacuum worker must have only PROC_IN_VACUUM flag since we
@@ -4186,6 +4218,14 @@ parallel_vacuum_main(dsm_segment *seg, shm_toc *toc)
*/
rel = table_open(lvshared->relid, ShareUpdateExclusiveLock);
+ /*
+ * Track progress of current index being vacuumed
+ */
+ pgstat_progress_start_command(PROGRESS_COMMAND_VACUUM,
+ RelationGetRelid(rel));
+
+ pgstat_progress_update_param(PROGRESS_VACUUM_LEADER_PID, leader->pid);
+
/*
* Open all indexes. indrels are sorted in order by OID, which should be
* matched to the leader's one.
@@ -4252,6 +4292,7 @@ parallel_vacuum_main(dsm_segment *seg, shm_toc *toc)
vac_close_indexes(nindexes, indrels, RowExclusiveLock);
table_close(rel, ShareUpdateExclusiveLock);
FreeAccessStrategy(vacrel.bstrategy);
+ pgstat_progress_end_command();
pfree(vacrel.indstats);
}
diff --git a/src/backend/access/nbtree/nbtree.c b/src/backend/access/nbtree/nbtree.c
index 40ad0956e0..e90fd77b86 100644
--- a/src/backend/access/nbtree/nbtree.c
+++ b/src/backend/access/nbtree/nbtree.c
@@ -996,10 +996,18 @@ btvacuumscan(IndexVacuumInfo *info, IndexBulkDeleteResult *stats,
btvacuumpage(&vstate, scanblkno);
if (info->report_progress)
pgstat_progress_update_param(PROGRESS_SCAN_BLOCKS_DONE,
- scanblkno);
+ scanblkno + 1);
}
}
+ if (info->report_progress)
+ {
+ pgstat_progress_update_param(PROGRESS_SCAN_BLOCKS_DONE,
+ 0);
+ pgstat_progress_update_param(PROGRESS_SCAN_BLOCKS_TOTAL,
+ 0);
+ }
+
/* Set statistics num_pages field to final size of index */
stats->num_pages = num_pages;
diff --git a/src/backend/access/spgist/spgvacuum.c b/src/backend/access/spgist/spgvacuum.c
index 76fb0374c4..19768f11ce 100644
--- a/src/backend/access/spgist/spgvacuum.c
+++ b/src/backend/access/spgist/spgvacuum.c
@@ -27,6 +27,8 @@
#include "storage/indexfsm.h"
#include "storage/lmgr.h"
#include "utils/snapmgr.h"
+#include "commands/progress.h"
+#include "pgstat.h"
/* Entry in pending-list of TIDs we need to revisit */
@@ -797,6 +799,7 @@ spgvacuumscan(spgBulkDeleteState *bds)
bool needLock;
BlockNumber num_pages,
blkno;
+ int blocks_scanned = 0;
/* Finish setting up spgBulkDeleteState */
initSpGistState(&bds->spgstate, index);
@@ -836,6 +839,11 @@ spgvacuumscan(spgBulkDeleteState *bds)
/* Quit if we've scanned the whole relation */
if (blkno >= num_pages)
break;
+
+ if (bds->info->report_progress)
+ pgstat_progress_update_param(PROGRESS_SCAN_BLOCKS_TOTAL,
+ num_pages);
+
/* Iterate over pages, then loop back to recheck length */
for (; blkno < num_pages; blkno++)
{
@@ -843,9 +851,22 @@ spgvacuumscan(spgBulkDeleteState *bds)
/* empty the pending-list after each page */
if (bds->pendingList != NULL)
spgprocesspending(bds);
+
+ blocks_scanned++;
+ if (bds->info->report_progress)
+ pgstat_progress_update_param(PROGRESS_SCAN_BLOCKS_DONE,
+ blocks_scanned + 1);
}
}
+ if (bds->info->report_progress)
+ {
+ pgstat_progress_update_param(PROGRESS_SCAN_BLOCKS_DONE,
+ 0);
+ pgstat_progress_update_param(PROGRESS_SCAN_BLOCKS_TOTAL,
+ 0);
+ }
+
/* Propagate local lastUsedPages cache to metablock */
SpGistUpdateMetaPage(index);
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index eb560955cd..953d238925 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1124,7 +1124,10 @@ CREATE VIEW pg_stat_progress_vacuum AS
END AS phase,
S.param2 AS heap_blks_total, S.param3 AS heap_blks_scanned,
S.param4 AS heap_blks_vacuumed, S.param5 AS index_vacuum_count,
- S.param6 AS max_dead_tuples, S.param7 AS num_dead_tuples
+ S.param6 AS max_dead_tuples, S.param7 AS num_dead_tuples,
+ S.param8 AS indrelid, S.param16 index_blks_total,
+ S.param17 AS index_blks_scanned,
+ CASE WHEN S.param9 = S.pid THEN NULL ELSE S.param9 END AS leader_pid
FROM pg_stat_get_progress_info('VACUUM') AS S
LEFT JOIN pg_database D ON S.datid = D.oid;
diff --git a/src/include/access/hash.h b/src/include/access/hash.h
index 1cce865be2..c00bb76e3e 100644
--- a/src/include/access/hash.h
+++ b/src/include/access/hash.h
@@ -478,6 +478,7 @@ extern void hashbucketcleanup(Relation rel, Bucket cur_bucket,
uint32 maxbucket, uint32 highmask, uint32 lowmask,
double *tuples_removed, double *num_index_tuples,
bool split_cleanup,
- IndexBulkDeleteCallback callback, void *callback_state);
+ IndexBulkDeleteCallback callback, void *callback_state,
+ bool report_progress, int *bucket_blocks_scanned);
#endif /* HASH_H */
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index d7bf16368b..4387a7c1f1 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -25,6 +25,8 @@
#define PROGRESS_VACUUM_NUM_INDEX_VACUUMS 4
#define PROGRESS_VACUUM_MAX_DEAD_TUPLES 5
#define PROGRESS_VACUUM_NUM_DEAD_TUPLES 6
+#define PROGRESS_VACUUM_CURRENT_INDRELID 7
+#define PROGRESS_VACUUM_LEADER_PID 8
/* Phases of vacuum (as advertised via PROGRESS_VACUUM_PHASE) */
#define PROGRESS_VACUUM_PHASE_SCAN_HEAP 1
^ permalink raw reply [nested|flat] 44+ messages in thread
* Re: Add index scan progress to pg_stat_progress_vacuum
@ 2021-12-15 22:09 Bossart, Nathan <[email protected]>
parent: Imseih (AWS), Sami <[email protected]>
2 siblings, 2 replies; 44+ messages in thread
From: Bossart, Nathan @ 2021-12-15 22:09 UTC (permalink / raw)
To: Imseih (AWS), Sami <[email protected]>; pgsql-hackers
On 12/1/21, 3:02 PM, "Imseih (AWS), Sami" <[email protected]> wrote:
> The current implementation of pg_stat_progress_vacuum does not
> provide progress on which index is being vacuumed making it
> difficult for a user to determine if the "vacuuming indexes" phase
> is making progress. By exposing which index is being scanned as well
> as the total progress the scan has made for the current cycle, a
> user can make better estimations on when the vacuum will complete.
+1
> The proposed patch adds 4 new columns to pg_stat_progress_vacuum:
>
> 1. indrelid - the relid of the index being vacuumed
> 2. index_blks_total - total number of blocks to be scanned in the
> current cycle
> 3. index_blks_scanned - number of blocks scanned in the current
> cycle
> 4. leader_pid - if the pid for the pg_stat_progress_vacuum entry is
> a leader or a vacuum worker. This patch places an entry for every
> worker pid ( if parallel ) as well as the leader pid
nitpick: Shouldn't index_blks_scanned be index_blks_vacuumed? IMO it
is more analogous to heap_blks_vacuumed.
This will tell us which indexes are currently being vacuumed and the
current progress of those operations, but it doesn't tell us which
indexes have already been vacuumed or which ones are pending vacuum.
I think such information is necessary to truly understand the current
progress of vacuuming indexes, and I can think of a couple of ways we
might provide it:
1. Make the new columns you've proposed return arrays. This isn't
very clean, but it would keep all the information for a given
vacuum operation in a single row. The indrelids column would be
populated with all the indexes that have been vacuumed, need to
be vacuumed, or are presently being vacuumed. The other index-
related columns would then have the associated stats and the
worker PID (which might be the same as the pid column depending
on whether parallel index vacuum was being done). Alternatively,
the index column could have an array of records, each containing
all the information for a given index.
2. Create a new view for just index vacuum progress information.
This would have similar information as 1. There would be an
entry for each index that has been vacuumed, needs to be
vacuumed, or is currently being vacuumed. And there would be an
easy way to join with pg_stat_progress_vacuum (e.g., leader_pid,
which again might be the same as our index vacuum PID depending
on whether we were doing parallel index vacuum). Note that it
would be possible for the PID of these entries to be null before
and after we process the index.
3. Instead of adding columns to pg_stat_progress_vacuum, adjust the
current ones to be more general, and then add new entries for
each of the indexes that have been, need to be, or currently are
being vacuumed. This is the most similar option to your current
proposal, but instead of introducing a column like
index_blks_total, we'd rename heap_blks_total to blks_total and
use that for both the heap and indexes. I think we'd still want
to add a leader_pid column. Again, we have to be prepared for
the PID to be null in this case. Or we could just make the pid
column always refer to the leader, and we could introduce a
worker_pid column. That might create confusion, though.
I wish option #1 was cleaner, because I think it would be really nice
to have all this information in a single row. However, I don't expect
much support for a 3-dimensional view, so I suspect option #2
(creating a separate view for index vacuum progress) is the way to go.
The other benefit of option #2 versus option #3 or your original
proposal is that it cleanly separates the top-level vacuum operations
and the index vacuum operations, which are related at the moment, but
which might not always be tied so closely together.
Nathan
^ permalink raw reply [nested|flat] 44+ messages in thread
* Re: Add index scan progress to pg_stat_progress_vacuum
@ 2021-12-16 21:37 Imseih (AWS), Sami <[email protected]>
parent: Bossart, Nathan <[email protected]>
1 sibling, 2 replies; 44+ messages in thread
From: Imseih (AWS), Sami @ 2021-12-16 21:37 UTC (permalink / raw)
To: Bossart, Nathan <[email protected]>; pgsql-hackers
On 12/15/21, 4:10 PM, "Bossart, Nathan" <[email protected]> wrote:
On 12/1/21, 3:02 PM, "Imseih (AWS), Sami" <[email protected]> wrote:
> The current implementation of pg_stat_progress_vacuum does not
> provide progress on which index is being vacuumed making it
> difficult for a user to determine if the "vacuuming indexes" phase
> is making progress. By exposing which index is being scanned as well
> as the total progress the scan has made for the current cycle, a
> user can make better estimations on when the vacuum will complete.
+1
> The proposed patch adds 4 new columns to pg_stat_progress_vacuum:
>
> 1. indrelid - the relid of the index being vacuumed
> 2. index_blks_total - total number of blocks to be scanned in the
> current cycle
> 3. index_blks_scanned - number of blocks scanned in the current
> cycle
> 4. leader_pid - if the pid for the pg_stat_progress_vacuum entry is
> a leader or a vacuum worker. This patch places an entry for every
> worker pid ( if parallel ) as well as the leader pid
nitpick: Shouldn't index_blks_scanned be index_blks_vacuumed? IMO it
is more analogous to heap_blks_vacuumed.
No, What is being tracked is the number of index blocks scanned from the total index blocks. The block will be scanned regardless if it will be vacuumed or not.
This will tell us which indexes are currently being vacuumed and the
current progress of those operations, but it doesn't tell us which
indexes have already been vacuumed or which ones are pending vacuum.
I think such information is necessary to truly understand the current
progress of vacuuming indexes, and I can think of a couple of ways we
might provide it:
1. Make the new columns you've proposed return arrays. This isn't
very clean, but it would keep all the information for a given
vacuum operation in a single row. The indrelids column would be
populated with all the indexes that have been vacuumed, need to
be vacuumed, or are presently being vacuumed. The other index-
related columns would then have the associated stats and the
worker PID (which might be the same as the pid column depending
on whether parallel index vacuum was being done). Alternatively,
the index column could have an array of records, each containing
all the information for a given index.
2. Create a new view for just index vacuum progress information.
This would have similar information as 1. There would be an
entry for each index that has been vacuumed, needs to be
vacuumed, or is currently being vacuumed. And there would be an
easy way to join with pg_stat_progress_vacuum (e.g., leader_pid,
which again might be the same as our index vacuum PID depending
on whether we were doing parallel index vacuum). Note that it
would be possible for the PID of these entries to be null before
and after we process the index.
3. Instead of adding columns to pg_stat_progress_vacuum, adjust the
current ones to be more general, and then add new entries for
each of the indexes that have been, need to be, or currently are
being vacuumed. This is the most similar option to your current
proposal, but instead of introducing a column like
index_blks_total, we'd rename heap_blks_total to blks_total and
use that for both the heap and indexes. I think we'd still want
to add a leader_pid column. Again, we have to be prepared for
the PID to be null in this case. Or we could just make the pid
column always refer to the leader, and we could introduce a
worker_pid column. That might create confusion, though.
I wish option #1 was cleaner, because I think it would be really nice
to have all this information in a single row. However, I don't expect
much support for a 3-dimensional view, so I suspect option #2
(creating a separate view for index vacuum progress) is the way to go.
The other benefit of option #2 versus option #3 or your original
proposal is that it cleanly separates the top-level vacuum operations
and the index vacuum operations, which are related at the moment, but
which might not always be tied so closely together.
Option #1 is not clean as you will need to unnest the array to make sense out of it. It will be too complex to use.
Option #3 I am reluctant to spent time looking at this option. It's more valuable to see progress per index instead of total.
Option #2 was one that I originally designed but backed away as it was introducing a new view. Thinking about it a bit more, this is a cleaner approach.
1. Having a view called pg_stat_progress_vacuum_worker to join with pg_stat_progress_vacuum is clean
2. No changes required to pg_stat_progress_vacuum
3. I’ll lean towards calling the view " pg_stat_progress_vacuum_worker" instead of " pg_stat_progress_vacuum_index", to perhaps allow us to track other items a vacuum worker may do in future releases. As of now, only indexes are vacuumed by workers.
I will rework the patch for option #2
Nathan
^ permalink raw reply [nested|flat] 44+ messages in thread
* Re: Add index scan progress to pg_stat_progress_vacuum
@ 2021-12-16 22:03 Greg Stark <[email protected]>
parent: Imseih (AWS), Sami <[email protected]>
1 sibling, 0 replies; 44+ messages in thread
From: Greg Stark @ 2021-12-16 22:03 UTC (permalink / raw)
To: Imseih (AWS), Sami <[email protected]>; +Cc: Bossart, Nathan <[email protected]>; pgsql-hackers
I had a similar question. And I'm still not clear from the response
what exactly index_blks_total is and whether it addresses it.
I think I agree that a user is likely to want to see the progress in a
way they can understand which means for a single index at a time.
I think what you're describing is that index_blks_total and
index_blks_scanned are the totals across all the indexes? That isn't
clear from the definitions but if that's what you intend then I think
that would work.
(For what it's worth what I was imagining was having a pair of
counters for blocks scanned and max blocks in this index and a second
counter for number of indexes processed and max number of indexes. But
I don't think that's necessarily any better than what you have)
^ permalink raw reply [nested|flat] 44+ messages in thread
* Re: Add index scan progress to pg_stat_progress_vacuum
@ 2021-12-20 17:55 Imseih (AWS), Sami <[email protected]>
parent: Imseih (AWS), Sami <[email protected]>
1 sibling, 0 replies; 44+ messages in thread
From: Imseih (AWS), Sami @ 2021-12-20 17:55 UTC (permalink / raw)
To: Bossart, Nathan <[email protected]>; pgsql-hackers
Here is a V2 attempt of the patch to include a new view called pg_stat_progress_vacuum_worker. Also, scans for index cleanups will also have an entry in the new view.
- here is the new view which reports an entry for every worker ( or leader ) that is doing index vacuum/index cleanup work.
postgres=# select * from pg_stat_progress_vacuum_worker ;
-[ RECORD 1 ]------+------
pid | 29355
leader_pid | 26501
indrelid | 16391
index_blks_total | 68894
index_blks_scanned | 35618
- the view can be joined with pg_stat_progress_vacuum. Sample output below
postgres=# select a.*, b.phase, b.heap_blks_total, b.heap_blks_scanned from pg_stat_progress_vacuum_worker a full outer join pg_stat_progress_vacuum b on a.pid = b.pid ;
pid | leader_pid | indrelid | index_blks_total | index_blks_scanned | phase | heap_blks_total | heap_blks_scanned
-------+------------+----------+------------------+--------------------+---------------------+-----------------+-------------------
26667 | 26667 | 16391 | 9165 | 401 | cleaning up indexes | 20082 | 20082
(1 row)
postgres=# select a.*, b.phase, b.heap_blks_total, b.heap_blks_scanned from pg_stat_progress_vacuum_worker a full outer join pg_stat_progress_vacuum b on a.pid = b.pid ;
-[ RECORD 1 ]------+------------------
pid | 26501
leader_pid | 26501
indrelid | 16393
index_blks_total | 145107
index_blks_scanned | 11060
phase | vacuuming indexes
heap_blks_total | 165375
heap_blks_scanned | 165375
-[ RECORD 2 ]------+------------------
pid | 28982
leader_pid | 26501
indrelid | 16392
index_blks_total | 47616
index_blks_scanned | 11861
phase | vacuuming indexes
heap_blks_total | 0
heap_blks_scanned | 0
-[ RECORD 3 ]------+------------------
pid | 28983
leader_pid | 26501
indrelid | 16391
index_blks_total | 56936
index_blks_scanned | 9138
phase | vacuuming indexes
heap_blks_total | 0
heap_blks_scanned | 0
On 12/15/21, 4:10 PM, "Bossart, Nathan" <[email protected]> wrote:
On 12/1/21, 3:02 PM, "Imseih (AWS), Sami" <[email protected]> wrote:
> The current implementation of pg_stat_progress_vacuum does not
> provide progress on which index is being vacuumed making it
> difficult for a user to determine if the "vacuuming indexes" phase
> is making progress. By exposing which index is being scanned as well
> as the total progress the scan has made for the current cycle, a
> user can make better estimations on when the vacuum will complete.
+1
> The proposed patch adds 4 new columns to pg_stat_progress_vacuum:
>
> 1. indrelid - the relid of the index being vacuumed
> 2. index_blks_total - total number of blocks to be scanned in the
> current cycle
> 3. index_blks_scanned - number of blocks scanned in the current
> cycle
> 4. leader_pid - if the pid for the pg_stat_progress_vacuum entry is
> a leader or a vacuum worker. This patch places an entry for every
> worker pid ( if parallel ) as well as the leader pid
nitpick: Shouldn't index_blks_scanned be index_blks_vacuumed? IMO it
is more analogous to heap_blks_vacuumed.
No, What is being tracked is the number of index blocks scanned from the total index blocks. The block will be scanned regardless if it will be vacuumed or not.
This will tell us which indexes are currently being vacuumed and the
current progress of those operations, but it doesn't tell us which
indexes have already been vacuumed or which ones are pending vacuum.
I think such information is necessary to truly understand the current
progress of vacuuming indexes, and I can think of a couple of ways we
might provide it:
1. Make the new columns you've proposed return arrays. This isn't
very clean, but it would keep all the information for a given
vacuum operation in a single row. The indrelids column would be
populated with all the indexes that have been vacuumed, need to
be vacuumed, or are presently being vacuumed. The other index-
related columns would then have the associated stats and the
worker PID (which might be the same as the pid column depending
on whether parallel index vacuum was being done). Alternatively,
the index column could have an array of records, each containing
all the information for a given index.
2. Create a new view for just index vacuum progress information.
This would have similar information as 1. There would be an
entry for each index that has been vacuumed, needs to be
vacuumed, or is currently being vacuumed. And there would be an
easy way to join with pg_stat_progress_vacuum (e.g., leader_pid,
which again might be the same as our index vacuum PID depending
on whether we were doing parallel index vacuum). Note that it
would be possible for the PID of these entries to be null before
and after we process the index.
3. Instead of adding columns to pg_stat_progress_vacuum, adjust the
current ones to be more general, and then add new entries for
each of the indexes that have been, need to be, or currently are
being vacuumed. This is the most similar option to your current
proposal, but instead of introducing a column like
index_blks_total, we'd rename heap_blks_total to blks_total and
use that for both the heap and indexes. I think we'd still want
to add a leader_pid column. Again, we have to be prepared for
the PID to be null in this case. Or we could just make the pid
column always refer to the leader, and we could introduce a
worker_pid column. That might create confusion, though.
I wish option #1 was cleaner, because I think it would be really nice
to have all this information in a single row. However, I don't expect
much support for a 3-dimensional view, so I suspect option #2
(creating a separate view for index vacuum progress) is the way to go.
The other benefit of option #2 versus option #3 or your original
proposal is that it cleanly separates the top-level vacuum operations
and the index vacuum operations, which are related at the moment, but
which might not always be tied so closely together.
Option #1 is not clean as you will need to unnest the array to make sense out of it. It will be too complex to use.
Option #3 I am reluctant to spent time looking at this option. It's more valuable to see progress per index instead of total.
Option #2 was one that I originally designed but backed away as it was introducing a new view. Thinking about it a bit more, this is a cleaner approach.
1. Having a view called pg_stat_progress_vacuum_worker to join with pg_stat_progress_vacuum is clean
2. No changes required to pg_stat_progress_vacuum
3. I’ll lean towards calling the view " pg_stat_progress_vacuum_worker" instead of " pg_stat_progress_vacuum_index", to perhaps allow us to track other items a vacuum worker may do in future releases. As of now, only indexes are vacuumed by workers.
I will rework the patch for option #2
Nathan
diff --git a/src/backend/access/brin/brin.c b/src/backend/access/brin/brin.c
index f521bb9635..97b2f8bc13 100644
--- a/src/backend/access/brin/brin.c
+++ b/src/backend/access/brin/brin.c
@@ -39,6 +39,8 @@
#include "utils/index_selfuncs.h"
#include "utils/memutils.h"
#include "utils/rel.h"
+#include "commands/progress.h"
+#include "pgstat.h"
/*
@@ -77,7 +79,7 @@ static void brinsummarize(Relation index, Relation heapRel, BlockNumber pageRang
static void form_and_insert_tuple(BrinBuildState *state);
static void union_tuples(BrinDesc *bdesc, BrinMemTuple *a,
BrinTuple *b);
-static void brin_vacuum_scan(Relation idxrel, BufferAccessStrategy strategy);
+static void brin_vacuum_scan(Relation idxrel, BufferAccessStrategy strategy, bool report_progress);
static bool add_values_to_range(Relation idxRel, BrinDesc *bdesc,
BrinMemTuple *dtup, Datum *values, bool *nulls);
static bool check_null_keys(BrinValues *bval, ScanKey *nullkeys, int nnullkeys);
@@ -953,7 +955,7 @@ brinvacuumcleanup(IndexVacuumInfo *info, IndexBulkDeleteResult *stats)
heapRel = table_open(IndexGetRelation(RelationGetRelid(info->index), false),
AccessShareLock);
- brin_vacuum_scan(info->index, info->strategy);
+ brin_vacuum_scan(info->index, info->strategy, info->report_progress);
brinsummarize(info->index, heapRel, BRIN_ALL_BLOCKRANGES, false,
&stats->num_index_tuples, &stats->num_index_tuples);
@@ -1635,16 +1637,24 @@ union_tuples(BrinDesc *bdesc, BrinMemTuple *a, BrinTuple *b)
* and such.
*/
static void
-brin_vacuum_scan(Relation idxrel, BufferAccessStrategy strategy)
+brin_vacuum_scan(Relation idxrel, BufferAccessStrategy strategy, bool report_progress)
{
BlockNumber nblocks;
BlockNumber blkno;
+ const int initprog_index[] = {
+ PROGRESS_SCAN_BLOCKS_DONE,
+ PROGRESS_SCAN_BLOCKS_TOTAL
+ };
+ int64 initprog_val[2];
/*
* Scan the index in physical order, and clean up any possible mess in
* each page.
*/
nblocks = RelationGetNumberOfBlocks(idxrel);
+ if (report_progress)
+ pgstat_progress_update_param(PROGRESS_SCAN_BLOCKS_TOTAL,
+ nblocks);
for (blkno = 0; blkno < nblocks; blkno++)
{
Buffer buf;
@@ -1656,9 +1666,20 @@ brin_vacuum_scan(Relation idxrel, BufferAccessStrategy strategy)
brin_page_cleanup(idxrel, buf);
+ if (report_progress)
+ pgstat_progress_update_param(PROGRESS_SCAN_BLOCKS_DONE,
+ blkno + 1);
+
ReleaseBuffer(buf);
}
+ if (report_progress)
+ {
+ initprog_val[0] = 0;
+ initprog_val[1] = 0;
+ pgstat_progress_update_multi_param(2, initprog_index, initprog_val);
+ }
+
/*
* Update all upper pages in the index's FSM, as well. This ensures not
* only that we propagate leaf-page FSM updates made by brin_page_cleanup,
diff --git a/src/backend/access/gin/ginvacuum.c b/src/backend/access/gin/ginvacuum.c
index a276eb020b..714586040a 100644
--- a/src/backend/access/gin/ginvacuum.c
+++ b/src/backend/access/gin/ginvacuum.c
@@ -24,6 +24,8 @@
#include "storage/lmgr.h"
#include "storage/predicate.h"
#include "utils/memutils.h"
+#include "commands/progress.h"
+#include "pgstat.h"
struct GinVacuumState
{
@@ -571,6 +573,14 @@ ginbulkdelete(IndexVacuumInfo *info, IndexBulkDeleteResult *stats,
Buffer buffer;
BlockNumber rootOfPostingTree[BLCKSZ / (sizeof(IndexTupleData) + sizeof(ItemId))];
uint32 nRoot;
+ BlockNumber num_pages;
+ bool needLock;
+ int blocks_scanned = 0;
+ const int initprog_index[] = {
+ PROGRESS_SCAN_BLOCKS_DONE,
+ PROGRESS_SCAN_BLOCKS_TOTAL
+ };
+ int64 initprog_val[2];
gvs.tmpCxt = AllocSetContextCreate(CurrentMemoryContext,
"Gin vacuum temporary context",
@@ -635,6 +645,19 @@ ginbulkdelete(IndexVacuumInfo *info, IndexBulkDeleteResult *stats,
RBM_NORMAL, info->strategy);
}
+ needLock = !RELATION_IS_LOCAL(index);
+
+ /* Get the current relation length */
+ if (needLock)
+ LockRelationForExtension(index, ExclusiveLock);
+ num_pages = RelationGetNumberOfBlocks(index);
+ if (needLock)
+ UnlockRelationForExtension(index, ExclusiveLock);
+
+ if (info->report_progress)
+ pgstat_progress_update_param(PROGRESS_SCAN_BLOCKS_TOTAL,
+ num_pages);
+
/* right now we found leftmost page in entry's BTree */
for (;;)
@@ -676,9 +699,20 @@ ginbulkdelete(IndexVacuumInfo *info, IndexBulkDeleteResult *stats,
buffer = ReadBufferExtended(index, MAIN_FORKNUM, blkno,
RBM_NORMAL, info->strategy);
+ blocks_scanned++;
+ if (info->report_progress)
+ pgstat_progress_update_param(PROGRESS_SCAN_BLOCKS_DONE,
+ blocks_scanned + 1);
LockBuffer(buffer, GIN_EXCLUSIVE);
}
+ if (info->report_progress)
+ {
+ initprog_val[0] = 0;
+ initprog_val[1] = 0;
+ pgstat_progress_update_multi_param(2, initprog_index, initprog_val);
+ }
+
MemoryContextDelete(gvs.tmpCxt);
return gvs.result;
@@ -694,6 +728,12 @@ ginvacuumcleanup(IndexVacuumInfo *info, IndexBulkDeleteResult *stats)
BlockNumber totFreePages;
GinState ginstate;
GinStatsData idxStat;
+ int blocks_scanned = 0;
+ const int initprog_index[] = {
+ PROGRESS_SCAN_BLOCKS_DONE,
+ PROGRESS_SCAN_BLOCKS_TOTAL
+ };
+ int64 initprog_val[2];
/*
* In an autovacuum analyze, we want to clean up pending insertions.
@@ -744,6 +784,9 @@ ginvacuumcleanup(IndexVacuumInfo *info, IndexBulkDeleteResult *stats)
totFreePages = 0;
+ if (info->report_progress)
+ pgstat_progress_update_param(PROGRESS_SCAN_BLOCKS_TOTAL,
+ npages);
for (blkno = GIN_ROOT_BLKNO; blkno < npages; blkno++)
{
Buffer buffer;
@@ -774,9 +817,21 @@ ginvacuumcleanup(IndexVacuumInfo *info, IndexBulkDeleteResult *stats)
idxStat.nEntries += PageGetMaxOffsetNumber(page);
}
+ blocks_scanned++;
+ if (info->report_progress)
+ pgstat_progress_update_param(PROGRESS_SCAN_BLOCKS_DONE,
+ blocks_scanned + 1);
+
UnlockReleaseBuffer(buffer);
}
+ if (info->report_progress)
+ {
+ initprog_val[0] = 0;
+ initprog_val[1] = 0;
+ pgstat_progress_update_multi_param(2, initprog_index, initprog_val);
+ }
+
/* Update the metapage with accurate page and entry counts */
idxStat.nTotalPages = npages;
ginUpdateStats(info->index, &idxStat, false);
diff --git a/src/backend/access/gist/gistvacuum.c b/src/backend/access/gist/gistvacuum.c
index 0663193531..e7d13c9eb6 100644
--- a/src/backend/access/gist/gistvacuum.c
+++ b/src/backend/access/gist/gistvacuum.c
@@ -23,6 +23,8 @@
#include "storage/indexfsm.h"
#include "storage/lmgr.h"
#include "utils/memutils.h"
+#include "commands/progress.h"
+#include "pgstat.h"
/* Working state needed by gistbulkdelete */
typedef struct
@@ -131,6 +133,11 @@ gistvacuumscan(IndexVacuumInfo *info, IndexBulkDeleteResult *stats,
bool needLock;
BlockNumber blkno;
MemoryContext oldctx;
+ const int initprog_index[] = {
+ PROGRESS_SCAN_BLOCKS_DONE,
+ PROGRESS_SCAN_BLOCKS_TOTAL
+ };
+ int64 initprog_val[2];
/*
* Reset fields that track information about the entire index now. This
@@ -215,9 +222,26 @@ gistvacuumscan(IndexVacuumInfo *info, IndexBulkDeleteResult *stats,
/* Quit if we've scanned the whole relation */
if (blkno >= num_pages)
break;
+
+ if (info->report_progress)
+ pgstat_progress_update_param(PROGRESS_SCAN_BLOCKS_TOTAL,
+ num_pages);
+
/* Iterate over pages, then loop back to recheck length */
for (; blkno < num_pages; blkno++)
+ {
gistvacuumpage(&vstate, blkno, blkno);
+ if (info->report_progress)
+ pgstat_progress_update_param(PROGRESS_SCAN_BLOCKS_DONE,
+ blkno + 1);
+ }
+ }
+
+ if (info->report_progress)
+ {
+ initprog_val[0] = 0;
+ initprog_val[1] = 0;
+ pgstat_progress_update_multi_param(2, initprog_index, initprog_val);
}
/*
diff --git a/src/backend/access/hash/hash.c b/src/backend/access/hash/hash.c
index 81c7da7ec6..2b4eed6aae 100644
--- a/src/backend/access/hash/hash.c
+++ b/src/backend/access/hash/hash.c
@@ -31,6 +31,7 @@
#include "utils/builtins.h"
#include "utils/index_selfuncs.h"
#include "utils/rel.h"
+#include "storage/lmgr.h"
/* Working state for hashbuild and its callback */
typedef struct
@@ -469,9 +470,21 @@ hashbulkdelete(IndexVacuumInfo *info, IndexBulkDeleteResult *stats,
Buffer metabuf = InvalidBuffer;
HashMetaPage metap;
HashMetaPage cachedmetap;
+ int blocks_scanned;
+ int bucket_blocks_scanned;
+ BlockNumber num_pages;
+ bool needLock;
+ const int initprog_index[] = {
+ PROGRESS_SCAN_BLOCKS_DONE,
+ PROGRESS_SCAN_BLOCKS_TOTAL
+ };
+ int64 initprog_val[2];
+
tuples_removed = 0;
num_index_tuples = 0;
+ blocks_scanned = 0;
+ bucket_blocks_scanned = 0;
/*
* We need a copy of the metapage so that we can use its hashm_spares[]
@@ -489,6 +502,19 @@ hashbulkdelete(IndexVacuumInfo *info, IndexBulkDeleteResult *stats,
cur_bucket = 0;
cur_maxbucket = orig_maxbucket;
+ needLock = !RELATION_IS_LOCAL(rel);
+
+ /* Get the current relation length */
+ if (needLock)
+ LockRelationForExtension(rel, ExclusiveLock);
+ num_pages = RelationGetNumberOfBlocks(rel);
+ if (needLock)
+ UnlockRelationForExtension(rel, ExclusiveLock);
+
+ if (info->report_progress)
+ pgstat_progress_update_param(PROGRESS_SCAN_BLOCKS_TOTAL,
+ num_pages);
+
loop_top:
while (cur_bucket <= cur_maxbucket)
{
@@ -504,6 +530,7 @@ loop_top:
bucket_blkno = BUCKET_TO_BLKNO(cachedmetap, cur_bucket);
blkno = bucket_blkno;
+ blocks_scanned++;
/*
* We need to acquire a cleanup lock on the primary bucket page to out
@@ -550,10 +577,14 @@ loop_top:
cachedmetap->hashm_highmask,
cachedmetap->hashm_lowmask, &tuples_removed,
&num_index_tuples, split_cleanup,
- callback, callback_state);
+ callback, callback_state, info->report_progress,
+ &bucket_blocks_scanned);
_hash_dropbuf(rel, bucket_buf);
+ pgstat_progress_update_param(PROGRESS_SCAN_BLOCKS_DONE,
+ blocks_scanned + bucket_blocks_scanned);
+
/* Advance to next bucket */
cur_bucket++;
}
@@ -633,6 +664,13 @@ loop_top:
stats->tuples_removed += tuples_removed;
/* hashvacuumcleanup will fill in num_pages */
+ if (info->report_progress)
+ {
+ initprog_val[0] = 0;
+ initprog_val[1] = 0;
+ pgstat_progress_update_multi_param(2, initprog_index, initprog_val);
+ }
+
return stats;
}
@@ -686,7 +724,8 @@ hashbucketcleanup(Relation rel, Bucket cur_bucket, Buffer bucket_buf,
uint32 maxbucket, uint32 highmask, uint32 lowmask,
double *tuples_removed, double *num_index_tuples,
bool split_cleanup,
- IndexBulkDeleteCallback callback, void *callback_state)
+ IndexBulkDeleteCallback callback, void *callback_state,
+ bool report_progress, int *bucket_blocks_scanned)
{
BlockNumber blkno;
Buffer buf;
@@ -718,6 +757,8 @@ hashbucketcleanup(Relation rel, Bucket cur_bucket, Buffer bucket_buf,
page = BufferGetPage(buf);
opaque = (HashPageOpaque) PageGetSpecialPointer(page);
+ bucket_blocks_scanned++;
+
/* Scan each tuple in page */
maxoffno = PageGetMaxOffsetNumber(page);
for (offno = FirstOffsetNumber;
@@ -916,4 +957,5 @@ hashbucketcleanup(Relation rel, Bucket cur_bucket, Buffer bucket_buf,
bstrategy);
else
LockBuffer(bucket_buf, BUFFER_LOCK_UNLOCK);
+
}
diff --git a/src/backend/access/hash/hashpage.c b/src/backend/access/hash/hashpage.c
index 159646c7c3..538d153df7 100644
--- a/src/backend/access/hash/hashpage.c
+++ b/src/backend/access/hash/hashpage.c
@@ -758,7 +758,7 @@ restart_expand:
hashbucketcleanup(rel, old_bucket, buf_oblkno, start_oblkno, NULL,
maxbucket, highmask, lowmask, NULL, NULL, true,
- NULL, NULL);
+ NULL, NULL, NULL, NULL);
_hash_dropbuf(rel, buf_oblkno);
@@ -1326,7 +1326,7 @@ _hash_splitbucket(Relation rel,
hashbucketcleanup(rel, obucket, bucket_obuf,
BufferGetBlockNumber(bucket_obuf), NULL,
maxbucket, highmask, lowmask, NULL, NULL, true,
- NULL, NULL);
+ NULL, NULL, NULL, NULL);
}
else
{
diff --git a/src/backend/access/heap/vacuumlazy.c b/src/backend/access/heap/vacuumlazy.c
index db6becfed5..ca26d5344e 100644
--- a/src/backend/access/heap/vacuumlazy.c
+++ b/src/backend/access/heap/vacuumlazy.c
@@ -517,6 +517,9 @@ heap_vacuum_rel(Relation rel, VacuumParams *params,
pgstat_progress_start_command(PROGRESS_COMMAND_VACUUM,
RelationGetRelid(rel));
+ pgstat_progress_update_param(PROGRESS_VACUUM_LEADER_PID,
+ MyProcPid);
+
vacuum_set_xid_limits(rel,
params->freeze_min_age,
params->freeze_table_age,
@@ -3021,12 +3024,17 @@ lazy_vacuum_one_index(Relation indrel, IndexBulkDeleteResult *istat,
IndexVacuumInfo ivinfo;
PGRUsage ru0;
LVSavedErrInfo saved_err_info;
+ const int initprog_index[] = {
+ PROGRESS_VACUUM_PHASE,
+ PROGRESS_VACUUM_CURRENT_INDRELID
+ };
+ int64 initprog_val[2];
pg_rusage_init(&ru0);
ivinfo.index = indrel;
ivinfo.analyze_only = false;
- ivinfo.report_progress = false;
+ ivinfo.report_progress = true;
ivinfo.estimated_count = true;
ivinfo.message_level = elevel;
ivinfo.num_heap_tuples = reltuples;
@@ -3044,10 +3052,19 @@ lazy_vacuum_one_index(Relation indrel, IndexBulkDeleteResult *istat,
VACUUM_ERRCB_PHASE_VACUUM_INDEX,
InvalidBlockNumber, InvalidOffsetNumber);
+ /* Report that we're vacuuming the index, advertising the indrelid */
+ initprog_val[0] = PROGRESS_VACUUM_PHASE_VACUUM_INDEX;
+ initprog_val[1] = RelationGetRelid(indrel);
+ pgstat_progress_update_multi_param(2, initprog_index, initprog_val);
+
/* Do bulk deletion */
istat = index_bulk_delete(&ivinfo, istat, lazy_tid_reaped,
(void *) vacrel->dead_items);
+ /* Report that we're done vacuuming the index */
+ pgstat_progress_update_param(PROGRESS_VACUUM_CURRENT_INDRELID,
+ 0);
+
ereport(elevel,
(errmsg("scanned index \"%s\" to remove %d row versions",
vacrel->indname, vacrel->dead_items->num_items),
@@ -3078,12 +3095,17 @@ lazy_cleanup_one_index(Relation indrel, IndexBulkDeleteResult *istat,
IndexVacuumInfo ivinfo;
PGRUsage ru0;
LVSavedErrInfo saved_err_info;
+ const int initprog_index[] = {
+ PROGRESS_VACUUM_PHASE,
+ PROGRESS_VACUUM_CURRENT_INDRELID
+ };
+ int64 initprog_val[2];
pg_rusage_init(&ru0);
ivinfo.index = indrel;
ivinfo.analyze_only = false;
- ivinfo.report_progress = false;
+ ivinfo.report_progress = true;
ivinfo.estimated_count = estimated_count;
ivinfo.message_level = elevel;
@@ -3102,8 +3124,18 @@ lazy_cleanup_one_index(Relation indrel, IndexBulkDeleteResult *istat,
VACUUM_ERRCB_PHASE_INDEX_CLEANUP,
InvalidBlockNumber, InvalidOffsetNumber);
+ /* Report that we're cleaning the index, advertising the indrelid */
+ initprog_val[0] = PROGRESS_VACUUM_PHASE_INDEX_CLEANUP;
+ initprog_val[1] = RelationGetRelid(indrel);
+ pgstat_progress_update_multi_param(2, initprog_index, initprog_val);
+
istat = index_vacuum_cleanup(&ivinfo, istat);
+ /* Report that we're done cleaning the index */
+ initprog_val[0] = 0;
+ initprog_val[1] = 0;
+ pgstat_progress_update_multi_param(2, initprog_index, initprog_val);
+
if (istat)
{
ereport(elevel,
@@ -4145,6 +4177,7 @@ parallel_vacuum_main(dsm_segment *seg, shm_toc *toc)
char *sharedquery;
LVRelState vacrel;
ErrorContextCallback errcallback;
+ PGPROC *leader = MyProc->lockGroupLeader;
/*
* A parallel vacuum worker must have only PROC_IN_VACUUM flag since we
@@ -4170,6 +4203,14 @@ parallel_vacuum_main(dsm_segment *seg, shm_toc *toc)
*/
rel = table_open(lvshared->relid, ShareUpdateExclusiveLock);
+ /*
+ *Track progress of current index being vacuumed
+ */
+ pgstat_progress_start_command(PROGRESS_COMMAND_VACUUM,
+ RelationGetRelid(rel));
+
+ pgstat_progress_update_param(PROGRESS_VACUUM_LEADER_PID, leader->pid);
+
/*
* Open all indexes. indrels are sorted in order by OID, which should be
* matched to the leader's one.
@@ -4241,6 +4282,7 @@ parallel_vacuum_main(dsm_segment *seg, shm_toc *toc)
vac_close_indexes(nindexes, indrels, RowExclusiveLock);
table_close(rel, ShareUpdateExclusiveLock);
FreeAccessStrategy(vacrel.bstrategy);
+ pgstat_progress_end_command();
pfree(vacrel.indstats);
}
diff --git a/src/backend/access/nbtree/nbtree.c b/src/backend/access/nbtree/nbtree.c
index dfce06dc49..15c11fcab8 100644
--- a/src/backend/access/nbtree/nbtree.c
+++ b/src/backend/access/nbtree/nbtree.c
@@ -909,6 +909,11 @@ btvacuumscan(IndexVacuumInfo *info, IndexBulkDeleteResult *stats,
BlockNumber num_pages;
BlockNumber scanblkno;
bool needLock;
+ const int initprog_index[] = {
+ PROGRESS_SCAN_BLOCKS_DONE,
+ PROGRESS_SCAN_BLOCKS_TOTAL
+ };
+ int64 initprog_val[2];
/*
* Reset fields that track information about the entire index now. This
@@ -997,10 +1002,17 @@ btvacuumscan(IndexVacuumInfo *info, IndexBulkDeleteResult *stats,
btvacuumpage(&vstate, scanblkno);
if (info->report_progress)
pgstat_progress_update_param(PROGRESS_SCAN_BLOCKS_DONE,
- scanblkno);
+ scanblkno + 1);
}
}
+ if (info->report_progress)
+ {
+ initprog_val[0] = 0;
+ initprog_val[1] = 0;
+ pgstat_progress_update_multi_param(2, initprog_index, initprog_val);
+ }
+
/* Set statistics num_pages field to final size of index */
stats->num_pages = num_pages;
diff --git a/src/backend/access/spgist/spgvacuum.c b/src/backend/access/spgist/spgvacuum.c
index 76fb0374c4..c71345fceb 100644
--- a/src/backend/access/spgist/spgvacuum.c
+++ b/src/backend/access/spgist/spgvacuum.c
@@ -27,6 +27,8 @@
#include "storage/indexfsm.h"
#include "storage/lmgr.h"
#include "utils/snapmgr.h"
+#include "commands/progress.h"
+#include "pgstat.h"
/* Entry in pending-list of TIDs we need to revisit */
@@ -797,6 +799,12 @@ spgvacuumscan(spgBulkDeleteState *bds)
bool needLock;
BlockNumber num_pages,
blkno;
+ int blocks_scanned = 0;
+ const int initprog_index[] = {
+ PROGRESS_SCAN_BLOCKS_DONE,
+ PROGRESS_SCAN_BLOCKS_TOTAL
+ };
+ int64 initprog_val[2];
/* Finish setting up spgBulkDeleteState */
initSpGistState(&bds->spgstate, index);
@@ -836,6 +844,11 @@ spgvacuumscan(spgBulkDeleteState *bds)
/* Quit if we've scanned the whole relation */
if (blkno >= num_pages)
break;
+
+ if (bds->info->report_progress)
+ pgstat_progress_update_param(PROGRESS_SCAN_BLOCKS_TOTAL,
+ num_pages);
+
/* Iterate over pages, then loop back to recheck length */
for (; blkno < num_pages; blkno++)
{
@@ -843,9 +856,21 @@ spgvacuumscan(spgBulkDeleteState *bds)
/* empty the pending-list after each page */
if (bds->pendingList != NULL)
spgprocesspending(bds);
+
+ blocks_scanned++;
+ if (bds->info->report_progress)
+ pgstat_progress_update_param(PROGRESS_SCAN_BLOCKS_DONE,
+ blocks_scanned + 1);
}
}
+ if (bds->info->report_progress)
+ {
+ initprog_val[0] = 0;
+ initprog_val[1] = 0;
+ pgstat_progress_update_multi_param(2, initprog_index, initprog_val);
+ }
+
/* Propagate local lastUsedPages cache to metablock */
SpGistUpdateMetaPage(index);
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index 61b515cdb8..9e0dc39314 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1128,6 +1128,17 @@ CREATE VIEW pg_stat_progress_vacuum AS
FROM pg_stat_get_progress_info('VACUUM') AS S
LEFT JOIN pg_database D ON S.datid = D.oid;
+CREATE VIEW pg_stat_progress_vacuum_worker AS
+ SELECT
+ S.pid,
+ S.param9 leader_pid,
+ S.param8 AS indrelid,
+ S.param16 index_blks_total,
+ S.param17 AS index_blks_scanned
+ FROM pg_stat_get_progress_info('VACUUM') AS S
+ LEFT JOIN pg_database D ON S.datid = D.oid
+ WHERE S.param8 > 0;
+
CREATE VIEW pg_stat_progress_cluster AS
SELECT
S.pid AS pid,
diff --git a/src/include/access/hash.h b/src/include/access/hash.h
index 1cce865be2..c00bb76e3e 100644
--- a/src/include/access/hash.h
+++ b/src/include/access/hash.h
@@ -478,6 +478,7 @@ extern void hashbucketcleanup(Relation rel, Bucket cur_bucket,
uint32 maxbucket, uint32 highmask, uint32 lowmask,
double *tuples_removed, double *num_index_tuples,
bool split_cleanup,
- IndexBulkDeleteCallback callback, void *callback_state);
+ IndexBulkDeleteCallback callback, void *callback_state,
+ bool report_progress, int *bucket_blocks_scanned);
#endif /* HASH_H */
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index d7bf16368b..4387a7c1f1 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -25,6 +25,8 @@
#define PROGRESS_VACUUM_NUM_INDEX_VACUUMS 4
#define PROGRESS_VACUUM_MAX_DEAD_TUPLES 5
#define PROGRESS_VACUUM_NUM_DEAD_TUPLES 6
+#define PROGRESS_VACUUM_CURRENT_INDRELID 7
+#define PROGRESS_VACUUM_LEADER_PID 8
/* Phases of vacuum (as advertised via PROGRESS_VACUUM_PHASE) */
#define PROGRESS_VACUUM_PHASE_SCAN_HEAP 1
Attachments:
[text/plain] patch.v2.txt (20.6K, ../../[email protected]/2-patch.v2.txt)
download | inline diff:
diff --git a/src/backend/access/brin/brin.c b/src/backend/access/brin/brin.c
index f521bb9635..97b2f8bc13 100644
--- a/src/backend/access/brin/brin.c
+++ b/src/backend/access/brin/brin.c
@@ -39,6 +39,8 @@
#include "utils/index_selfuncs.h"
#include "utils/memutils.h"
#include "utils/rel.h"
+#include "commands/progress.h"
+#include "pgstat.h"
/*
@@ -77,7 +79,7 @@ static void brinsummarize(Relation index, Relation heapRel, BlockNumber pageRang
static void form_and_insert_tuple(BrinBuildState *state);
static void union_tuples(BrinDesc *bdesc, BrinMemTuple *a,
BrinTuple *b);
-static void brin_vacuum_scan(Relation idxrel, BufferAccessStrategy strategy);
+static void brin_vacuum_scan(Relation idxrel, BufferAccessStrategy strategy, bool report_progress);
static bool add_values_to_range(Relation idxRel, BrinDesc *bdesc,
BrinMemTuple *dtup, Datum *values, bool *nulls);
static bool check_null_keys(BrinValues *bval, ScanKey *nullkeys, int nnullkeys);
@@ -953,7 +955,7 @@ brinvacuumcleanup(IndexVacuumInfo *info, IndexBulkDeleteResult *stats)
heapRel = table_open(IndexGetRelation(RelationGetRelid(info->index), false),
AccessShareLock);
- brin_vacuum_scan(info->index, info->strategy);
+ brin_vacuum_scan(info->index, info->strategy, info->report_progress);
brinsummarize(info->index, heapRel, BRIN_ALL_BLOCKRANGES, false,
&stats->num_index_tuples, &stats->num_index_tuples);
@@ -1635,16 +1637,24 @@ union_tuples(BrinDesc *bdesc, BrinMemTuple *a, BrinTuple *b)
* and such.
*/
static void
-brin_vacuum_scan(Relation idxrel, BufferAccessStrategy strategy)
+brin_vacuum_scan(Relation idxrel, BufferAccessStrategy strategy, bool report_progress)
{
BlockNumber nblocks;
BlockNumber blkno;
+ const int initprog_index[] = {
+ PROGRESS_SCAN_BLOCKS_DONE,
+ PROGRESS_SCAN_BLOCKS_TOTAL
+ };
+ int64 initprog_val[2];
/*
* Scan the index in physical order, and clean up any possible mess in
* each page.
*/
nblocks = RelationGetNumberOfBlocks(idxrel);
+ if (report_progress)
+ pgstat_progress_update_param(PROGRESS_SCAN_BLOCKS_TOTAL,
+ nblocks);
for (blkno = 0; blkno < nblocks; blkno++)
{
Buffer buf;
@@ -1656,9 +1666,20 @@ brin_vacuum_scan(Relation idxrel, BufferAccessStrategy strategy)
brin_page_cleanup(idxrel, buf);
+ if (report_progress)
+ pgstat_progress_update_param(PROGRESS_SCAN_BLOCKS_DONE,
+ blkno + 1);
+
ReleaseBuffer(buf);
}
+ if (report_progress)
+ {
+ initprog_val[0] = 0;
+ initprog_val[1] = 0;
+ pgstat_progress_update_multi_param(2, initprog_index, initprog_val);
+ }
+
/*
* Update all upper pages in the index's FSM, as well. This ensures not
* only that we propagate leaf-page FSM updates made by brin_page_cleanup,
diff --git a/src/backend/access/gin/ginvacuum.c b/src/backend/access/gin/ginvacuum.c
index a276eb020b..714586040a 100644
--- a/src/backend/access/gin/ginvacuum.c
+++ b/src/backend/access/gin/ginvacuum.c
@@ -24,6 +24,8 @@
#include "storage/lmgr.h"
#include "storage/predicate.h"
#include "utils/memutils.h"
+#include "commands/progress.h"
+#include "pgstat.h"
struct GinVacuumState
{
@@ -571,6 +573,14 @@ ginbulkdelete(IndexVacuumInfo *info, IndexBulkDeleteResult *stats,
Buffer buffer;
BlockNumber rootOfPostingTree[BLCKSZ / (sizeof(IndexTupleData) + sizeof(ItemId))];
uint32 nRoot;
+ BlockNumber num_pages;
+ bool needLock;
+ int blocks_scanned = 0;
+ const int initprog_index[] = {
+ PROGRESS_SCAN_BLOCKS_DONE,
+ PROGRESS_SCAN_BLOCKS_TOTAL
+ };
+ int64 initprog_val[2];
gvs.tmpCxt = AllocSetContextCreate(CurrentMemoryContext,
"Gin vacuum temporary context",
@@ -635,6 +645,19 @@ ginbulkdelete(IndexVacuumInfo *info, IndexBulkDeleteResult *stats,
RBM_NORMAL, info->strategy);
}
+ needLock = !RELATION_IS_LOCAL(index);
+
+ /* Get the current relation length */
+ if (needLock)
+ LockRelationForExtension(index, ExclusiveLock);
+ num_pages = RelationGetNumberOfBlocks(index);
+ if (needLock)
+ UnlockRelationForExtension(index, ExclusiveLock);
+
+ if (info->report_progress)
+ pgstat_progress_update_param(PROGRESS_SCAN_BLOCKS_TOTAL,
+ num_pages);
+
/* right now we found leftmost page in entry's BTree */
for (;;)
@@ -676,9 +699,20 @@ ginbulkdelete(IndexVacuumInfo *info, IndexBulkDeleteResult *stats,
buffer = ReadBufferExtended(index, MAIN_FORKNUM, blkno,
RBM_NORMAL, info->strategy);
+ blocks_scanned++;
+ if (info->report_progress)
+ pgstat_progress_update_param(PROGRESS_SCAN_BLOCKS_DONE,
+ blocks_scanned + 1);
LockBuffer(buffer, GIN_EXCLUSIVE);
}
+ if (info->report_progress)
+ {
+ initprog_val[0] = 0;
+ initprog_val[1] = 0;
+ pgstat_progress_update_multi_param(2, initprog_index, initprog_val);
+ }
+
MemoryContextDelete(gvs.tmpCxt);
return gvs.result;
@@ -694,6 +728,12 @@ ginvacuumcleanup(IndexVacuumInfo *info, IndexBulkDeleteResult *stats)
BlockNumber totFreePages;
GinState ginstate;
GinStatsData idxStat;
+ int blocks_scanned = 0;
+ const int initprog_index[] = {
+ PROGRESS_SCAN_BLOCKS_DONE,
+ PROGRESS_SCAN_BLOCKS_TOTAL
+ };
+ int64 initprog_val[2];
/*
* In an autovacuum analyze, we want to clean up pending insertions.
@@ -744,6 +784,9 @@ ginvacuumcleanup(IndexVacuumInfo *info, IndexBulkDeleteResult *stats)
totFreePages = 0;
+ if (info->report_progress)
+ pgstat_progress_update_param(PROGRESS_SCAN_BLOCKS_TOTAL,
+ npages);
for (blkno = GIN_ROOT_BLKNO; blkno < npages; blkno++)
{
Buffer buffer;
@@ -774,9 +817,21 @@ ginvacuumcleanup(IndexVacuumInfo *info, IndexBulkDeleteResult *stats)
idxStat.nEntries += PageGetMaxOffsetNumber(page);
}
+ blocks_scanned++;
+ if (info->report_progress)
+ pgstat_progress_update_param(PROGRESS_SCAN_BLOCKS_DONE,
+ blocks_scanned + 1);
+
UnlockReleaseBuffer(buffer);
}
+ if (info->report_progress)
+ {
+ initprog_val[0] = 0;
+ initprog_val[1] = 0;
+ pgstat_progress_update_multi_param(2, initprog_index, initprog_val);
+ }
+
/* Update the metapage with accurate page and entry counts */
idxStat.nTotalPages = npages;
ginUpdateStats(info->index, &idxStat, false);
diff --git a/src/backend/access/gist/gistvacuum.c b/src/backend/access/gist/gistvacuum.c
index 0663193531..e7d13c9eb6 100644
--- a/src/backend/access/gist/gistvacuum.c
+++ b/src/backend/access/gist/gistvacuum.c
@@ -23,6 +23,8 @@
#include "storage/indexfsm.h"
#include "storage/lmgr.h"
#include "utils/memutils.h"
+#include "commands/progress.h"
+#include "pgstat.h"
/* Working state needed by gistbulkdelete */
typedef struct
@@ -131,6 +133,11 @@ gistvacuumscan(IndexVacuumInfo *info, IndexBulkDeleteResult *stats,
bool needLock;
BlockNumber blkno;
MemoryContext oldctx;
+ const int initprog_index[] = {
+ PROGRESS_SCAN_BLOCKS_DONE,
+ PROGRESS_SCAN_BLOCKS_TOTAL
+ };
+ int64 initprog_val[2];
/*
* Reset fields that track information about the entire index now. This
@@ -215,9 +222,26 @@ gistvacuumscan(IndexVacuumInfo *info, IndexBulkDeleteResult *stats,
/* Quit if we've scanned the whole relation */
if (blkno >= num_pages)
break;
+
+ if (info->report_progress)
+ pgstat_progress_update_param(PROGRESS_SCAN_BLOCKS_TOTAL,
+ num_pages);
+
/* Iterate over pages, then loop back to recheck length */
for (; blkno < num_pages; blkno++)
+ {
gistvacuumpage(&vstate, blkno, blkno);
+ if (info->report_progress)
+ pgstat_progress_update_param(PROGRESS_SCAN_BLOCKS_DONE,
+ blkno + 1);
+ }
+ }
+
+ if (info->report_progress)
+ {
+ initprog_val[0] = 0;
+ initprog_val[1] = 0;
+ pgstat_progress_update_multi_param(2, initprog_index, initprog_val);
}
/*
diff --git a/src/backend/access/hash/hash.c b/src/backend/access/hash/hash.c
index 81c7da7ec6..2b4eed6aae 100644
--- a/src/backend/access/hash/hash.c
+++ b/src/backend/access/hash/hash.c
@@ -31,6 +31,7 @@
#include "utils/builtins.h"
#include "utils/index_selfuncs.h"
#include "utils/rel.h"
+#include "storage/lmgr.h"
/* Working state for hashbuild and its callback */
typedef struct
@@ -469,9 +470,21 @@ hashbulkdelete(IndexVacuumInfo *info, IndexBulkDeleteResult *stats,
Buffer metabuf = InvalidBuffer;
HashMetaPage metap;
HashMetaPage cachedmetap;
+ int blocks_scanned;
+ int bucket_blocks_scanned;
+ BlockNumber num_pages;
+ bool needLock;
+ const int initprog_index[] = {
+ PROGRESS_SCAN_BLOCKS_DONE,
+ PROGRESS_SCAN_BLOCKS_TOTAL
+ };
+ int64 initprog_val[2];
+
tuples_removed = 0;
num_index_tuples = 0;
+ blocks_scanned = 0;
+ bucket_blocks_scanned = 0;
/*
* We need a copy of the metapage so that we can use its hashm_spares[]
@@ -489,6 +502,19 @@ hashbulkdelete(IndexVacuumInfo *info, IndexBulkDeleteResult *stats,
cur_bucket = 0;
cur_maxbucket = orig_maxbucket;
+ needLock = !RELATION_IS_LOCAL(rel);
+
+ /* Get the current relation length */
+ if (needLock)
+ LockRelationForExtension(rel, ExclusiveLock);
+ num_pages = RelationGetNumberOfBlocks(rel);
+ if (needLock)
+ UnlockRelationForExtension(rel, ExclusiveLock);
+
+ if (info->report_progress)
+ pgstat_progress_update_param(PROGRESS_SCAN_BLOCKS_TOTAL,
+ num_pages);
+
loop_top:
while (cur_bucket <= cur_maxbucket)
{
@@ -504,6 +530,7 @@ loop_top:
bucket_blkno = BUCKET_TO_BLKNO(cachedmetap, cur_bucket);
blkno = bucket_blkno;
+ blocks_scanned++;
/*
* We need to acquire a cleanup lock on the primary bucket page to out
@@ -550,10 +577,14 @@ loop_top:
cachedmetap->hashm_highmask,
cachedmetap->hashm_lowmask, &tuples_removed,
&num_index_tuples, split_cleanup,
- callback, callback_state);
+ callback, callback_state, info->report_progress,
+ &bucket_blocks_scanned);
_hash_dropbuf(rel, bucket_buf);
+ pgstat_progress_update_param(PROGRESS_SCAN_BLOCKS_DONE,
+ blocks_scanned + bucket_blocks_scanned);
+
/* Advance to next bucket */
cur_bucket++;
}
@@ -633,6 +664,13 @@ loop_top:
stats->tuples_removed += tuples_removed;
/* hashvacuumcleanup will fill in num_pages */
+ if (info->report_progress)
+ {
+ initprog_val[0] = 0;
+ initprog_val[1] = 0;
+ pgstat_progress_update_multi_param(2, initprog_index, initprog_val);
+ }
+
return stats;
}
@@ -686,7 +724,8 @@ hashbucketcleanup(Relation rel, Bucket cur_bucket, Buffer bucket_buf,
uint32 maxbucket, uint32 highmask, uint32 lowmask,
double *tuples_removed, double *num_index_tuples,
bool split_cleanup,
- IndexBulkDeleteCallback callback, void *callback_state)
+ IndexBulkDeleteCallback callback, void *callback_state,
+ bool report_progress, int *bucket_blocks_scanned)
{
BlockNumber blkno;
Buffer buf;
@@ -718,6 +757,8 @@ hashbucketcleanup(Relation rel, Bucket cur_bucket, Buffer bucket_buf,
page = BufferGetPage(buf);
opaque = (HashPageOpaque) PageGetSpecialPointer(page);
+ bucket_blocks_scanned++;
+
/* Scan each tuple in page */
maxoffno = PageGetMaxOffsetNumber(page);
for (offno = FirstOffsetNumber;
@@ -916,4 +957,5 @@ hashbucketcleanup(Relation rel, Bucket cur_bucket, Buffer bucket_buf,
bstrategy);
else
LockBuffer(bucket_buf, BUFFER_LOCK_UNLOCK);
+
}
diff --git a/src/backend/access/hash/hashpage.c b/src/backend/access/hash/hashpage.c
index 159646c7c3..538d153df7 100644
--- a/src/backend/access/hash/hashpage.c
+++ b/src/backend/access/hash/hashpage.c
@@ -758,7 +758,7 @@ restart_expand:
hashbucketcleanup(rel, old_bucket, buf_oblkno, start_oblkno, NULL,
maxbucket, highmask, lowmask, NULL, NULL, true,
- NULL, NULL);
+ NULL, NULL, NULL, NULL);
_hash_dropbuf(rel, buf_oblkno);
@@ -1326,7 +1326,7 @@ _hash_splitbucket(Relation rel,
hashbucketcleanup(rel, obucket, bucket_obuf,
BufferGetBlockNumber(bucket_obuf), NULL,
maxbucket, highmask, lowmask, NULL, NULL, true,
- NULL, NULL);
+ NULL, NULL, NULL, NULL);
}
else
{
diff --git a/src/backend/access/heap/vacuumlazy.c b/src/backend/access/heap/vacuumlazy.c
index db6becfed5..ca26d5344e 100644
--- a/src/backend/access/heap/vacuumlazy.c
+++ b/src/backend/access/heap/vacuumlazy.c
@@ -517,6 +517,9 @@ heap_vacuum_rel(Relation rel, VacuumParams *params,
pgstat_progress_start_command(PROGRESS_COMMAND_VACUUM,
RelationGetRelid(rel));
+ pgstat_progress_update_param(PROGRESS_VACUUM_LEADER_PID,
+ MyProcPid);
+
vacuum_set_xid_limits(rel,
params->freeze_min_age,
params->freeze_table_age,
@@ -3021,12 +3024,17 @@ lazy_vacuum_one_index(Relation indrel, IndexBulkDeleteResult *istat,
IndexVacuumInfo ivinfo;
PGRUsage ru0;
LVSavedErrInfo saved_err_info;
+ const int initprog_index[] = {
+ PROGRESS_VACUUM_PHASE,
+ PROGRESS_VACUUM_CURRENT_INDRELID
+ };
+ int64 initprog_val[2];
pg_rusage_init(&ru0);
ivinfo.index = indrel;
ivinfo.analyze_only = false;
- ivinfo.report_progress = false;
+ ivinfo.report_progress = true;
ivinfo.estimated_count = true;
ivinfo.message_level = elevel;
ivinfo.num_heap_tuples = reltuples;
@@ -3044,10 +3052,19 @@ lazy_vacuum_one_index(Relation indrel, IndexBulkDeleteResult *istat,
VACUUM_ERRCB_PHASE_VACUUM_INDEX,
InvalidBlockNumber, InvalidOffsetNumber);
+ /* Report that we're vacuuming the index, advertising the indrelid */
+ initprog_val[0] = PROGRESS_VACUUM_PHASE_VACUUM_INDEX;
+ initprog_val[1] = RelationGetRelid(indrel);
+ pgstat_progress_update_multi_param(2, initprog_index, initprog_val);
+
/* Do bulk deletion */
istat = index_bulk_delete(&ivinfo, istat, lazy_tid_reaped,
(void *) vacrel->dead_items);
+ /* Report that we're done vacuuming the index */
+ pgstat_progress_update_param(PROGRESS_VACUUM_CURRENT_INDRELID,
+ 0);
+
ereport(elevel,
(errmsg("scanned index \"%s\" to remove %d row versions",
vacrel->indname, vacrel->dead_items->num_items),
@@ -3078,12 +3095,17 @@ lazy_cleanup_one_index(Relation indrel, IndexBulkDeleteResult *istat,
IndexVacuumInfo ivinfo;
PGRUsage ru0;
LVSavedErrInfo saved_err_info;
+ const int initprog_index[] = {
+ PROGRESS_VACUUM_PHASE,
+ PROGRESS_VACUUM_CURRENT_INDRELID
+ };
+ int64 initprog_val[2];
pg_rusage_init(&ru0);
ivinfo.index = indrel;
ivinfo.analyze_only = false;
- ivinfo.report_progress = false;
+ ivinfo.report_progress = true;
ivinfo.estimated_count = estimated_count;
ivinfo.message_level = elevel;
@@ -3102,8 +3124,18 @@ lazy_cleanup_one_index(Relation indrel, IndexBulkDeleteResult *istat,
VACUUM_ERRCB_PHASE_INDEX_CLEANUP,
InvalidBlockNumber, InvalidOffsetNumber);
+ /* Report that we're cleaning the index, advertising the indrelid */
+ initprog_val[0] = PROGRESS_VACUUM_PHASE_INDEX_CLEANUP;
+ initprog_val[1] = RelationGetRelid(indrel);
+ pgstat_progress_update_multi_param(2, initprog_index, initprog_val);
+
istat = index_vacuum_cleanup(&ivinfo, istat);
+ /* Report that we're done cleaning the index */
+ initprog_val[0] = 0;
+ initprog_val[1] = 0;
+ pgstat_progress_update_multi_param(2, initprog_index, initprog_val);
+
if (istat)
{
ereport(elevel,
@@ -4145,6 +4177,7 @@ parallel_vacuum_main(dsm_segment *seg, shm_toc *toc)
char *sharedquery;
LVRelState vacrel;
ErrorContextCallback errcallback;
+ PGPROC *leader = MyProc->lockGroupLeader;
/*
* A parallel vacuum worker must have only PROC_IN_VACUUM flag since we
@@ -4170,6 +4203,14 @@ parallel_vacuum_main(dsm_segment *seg, shm_toc *toc)
*/
rel = table_open(lvshared->relid, ShareUpdateExclusiveLock);
+ /*
+ *Track progress of current index being vacuumed
+ */
+ pgstat_progress_start_command(PROGRESS_COMMAND_VACUUM,
+ RelationGetRelid(rel));
+
+ pgstat_progress_update_param(PROGRESS_VACUUM_LEADER_PID, leader->pid);
+
/*
* Open all indexes. indrels are sorted in order by OID, which should be
* matched to the leader's one.
@@ -4241,6 +4282,7 @@ parallel_vacuum_main(dsm_segment *seg, shm_toc *toc)
vac_close_indexes(nindexes, indrels, RowExclusiveLock);
table_close(rel, ShareUpdateExclusiveLock);
FreeAccessStrategy(vacrel.bstrategy);
+ pgstat_progress_end_command();
pfree(vacrel.indstats);
}
diff --git a/src/backend/access/nbtree/nbtree.c b/src/backend/access/nbtree/nbtree.c
index dfce06dc49..15c11fcab8 100644
--- a/src/backend/access/nbtree/nbtree.c
+++ b/src/backend/access/nbtree/nbtree.c
@@ -909,6 +909,11 @@ btvacuumscan(IndexVacuumInfo *info, IndexBulkDeleteResult *stats,
BlockNumber num_pages;
BlockNumber scanblkno;
bool needLock;
+ const int initprog_index[] = {
+ PROGRESS_SCAN_BLOCKS_DONE,
+ PROGRESS_SCAN_BLOCKS_TOTAL
+ };
+ int64 initprog_val[2];
/*
* Reset fields that track information about the entire index now. This
@@ -997,10 +1002,17 @@ btvacuumscan(IndexVacuumInfo *info, IndexBulkDeleteResult *stats,
btvacuumpage(&vstate, scanblkno);
if (info->report_progress)
pgstat_progress_update_param(PROGRESS_SCAN_BLOCKS_DONE,
- scanblkno);
+ scanblkno + 1);
}
}
+ if (info->report_progress)
+ {
+ initprog_val[0] = 0;
+ initprog_val[1] = 0;
+ pgstat_progress_update_multi_param(2, initprog_index, initprog_val);
+ }
+
/* Set statistics num_pages field to final size of index */
stats->num_pages = num_pages;
diff --git a/src/backend/access/spgist/spgvacuum.c b/src/backend/access/spgist/spgvacuum.c
index 76fb0374c4..c71345fceb 100644
--- a/src/backend/access/spgist/spgvacuum.c
+++ b/src/backend/access/spgist/spgvacuum.c
@@ -27,6 +27,8 @@
#include "storage/indexfsm.h"
#include "storage/lmgr.h"
#include "utils/snapmgr.h"
+#include "commands/progress.h"
+#include "pgstat.h"
/* Entry in pending-list of TIDs we need to revisit */
@@ -797,6 +799,12 @@ spgvacuumscan(spgBulkDeleteState *bds)
bool needLock;
BlockNumber num_pages,
blkno;
+ int blocks_scanned = 0;
+ const int initprog_index[] = {
+ PROGRESS_SCAN_BLOCKS_DONE,
+ PROGRESS_SCAN_BLOCKS_TOTAL
+ };
+ int64 initprog_val[2];
/* Finish setting up spgBulkDeleteState */
initSpGistState(&bds->spgstate, index);
@@ -836,6 +844,11 @@ spgvacuumscan(spgBulkDeleteState *bds)
/* Quit if we've scanned the whole relation */
if (blkno >= num_pages)
break;
+
+ if (bds->info->report_progress)
+ pgstat_progress_update_param(PROGRESS_SCAN_BLOCKS_TOTAL,
+ num_pages);
+
/* Iterate over pages, then loop back to recheck length */
for (; blkno < num_pages; blkno++)
{
@@ -843,9 +856,21 @@ spgvacuumscan(spgBulkDeleteState *bds)
/* empty the pending-list after each page */
if (bds->pendingList != NULL)
spgprocesspending(bds);
+
+ blocks_scanned++;
+ if (bds->info->report_progress)
+ pgstat_progress_update_param(PROGRESS_SCAN_BLOCKS_DONE,
+ blocks_scanned + 1);
}
}
+ if (bds->info->report_progress)
+ {
+ initprog_val[0] = 0;
+ initprog_val[1] = 0;
+ pgstat_progress_update_multi_param(2, initprog_index, initprog_val);
+ }
+
/* Propagate local lastUsedPages cache to metablock */
SpGistUpdateMetaPage(index);
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index 61b515cdb8..9e0dc39314 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1128,6 +1128,17 @@ CREATE VIEW pg_stat_progress_vacuum AS
FROM pg_stat_get_progress_info('VACUUM') AS S
LEFT JOIN pg_database D ON S.datid = D.oid;
+CREATE VIEW pg_stat_progress_vacuum_worker AS
+ SELECT
+ S.pid,
+ S.param9 leader_pid,
+ S.param8 AS indrelid,
+ S.param16 index_blks_total,
+ S.param17 AS index_blks_scanned
+ FROM pg_stat_get_progress_info('VACUUM') AS S
+ LEFT JOIN pg_database D ON S.datid = D.oid
+ WHERE S.param8 > 0;
+
CREATE VIEW pg_stat_progress_cluster AS
SELECT
S.pid AS pid,
diff --git a/src/include/access/hash.h b/src/include/access/hash.h
index 1cce865be2..c00bb76e3e 100644
--- a/src/include/access/hash.h
+++ b/src/include/access/hash.h
@@ -478,6 +478,7 @@ extern void hashbucketcleanup(Relation rel, Bucket cur_bucket,
uint32 maxbucket, uint32 highmask, uint32 lowmask,
double *tuples_removed, double *num_index_tuples,
bool split_cleanup,
- IndexBulkDeleteCallback callback, void *callback_state);
+ IndexBulkDeleteCallback callback, void *callback_state,
+ bool report_progress, int *bucket_blocks_scanned);
#endif /* HASH_H */
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index d7bf16368b..4387a7c1f1 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -25,6 +25,8 @@
#define PROGRESS_VACUUM_NUM_INDEX_VACUUMS 4
#define PROGRESS_VACUUM_MAX_DEAD_TUPLES 5
#define PROGRESS_VACUUM_NUM_DEAD_TUPLES 6
+#define PROGRESS_VACUUM_CURRENT_INDRELID 7
+#define PROGRESS_VACUUM_LEADER_PID 8
/* Phases of vacuum (as advertised via PROGRESS_VACUUM_PHASE) */
#define PROGRESS_VACUUM_PHASE_SCAN_HEAP 1
^ permalink raw reply [nested|flat] 44+ messages in thread
* [PATCH] Add index scan progress to pg_stat_progress_vacuum
@ 2021-12-20 17:55 Imseih (AWS), Sami <[email protected]>
0 siblings, 0 replies; 44+ messages in thread
From: Imseih (AWS), Sami @ 2021-12-20 17:55 UTC (permalink / raw)
Here is a V2 attempt of the patch to include a new view called pg_stat_progress_vacuum_worker. Also, scans for index cleanups will also have an entry in the new view.
Re: Add index scan progress to pg_stat_progress_vacuum
---
src/backend/access/brin/brin.c | 27 +++++++++++--
src/backend/access/gin/ginvacuum.c | 55 +++++++++++++++++++++++++++
src/backend/access/gist/gistvacuum.c | 24 ++++++++++++
src/backend/access/hash/hash.c | 46 +++++++++++++++++++++-
src/backend/access/hash/hashpage.c | 4 +-
src/backend/access/heap/vacuumlazy.c | 36 +++++++++++++++++-
src/backend/access/nbtree/nbtree.c | 14 ++++++-
src/backend/access/spgist/spgvacuum.c | 25 ++++++++++++
src/backend/catalog/system_views.sql | 11 ++++++
src/backend/commands/vacuumparallel.c | 11 ++++++
src/include/access/hash.h | 3 +-
src/include/commands/progress.h | 2 +
12 files changed, 247 insertions(+), 11 deletions(-)
diff --git a/src/backend/access/brin/brin.c b/src/backend/access/brin/brin.c
index f521bb96356..97b2f8bc13a 100644
--- a/src/backend/access/brin/brin.c
+++ b/src/backend/access/brin/brin.c
@@ -39,6 +39,8 @@
#include "utils/index_selfuncs.h"
#include "utils/memutils.h"
#include "utils/rel.h"
+#include "commands/progress.h"
+#include "pgstat.h"
/*
@@ -77,7 +79,7 @@ static void brinsummarize(Relation index, Relation heapRel, BlockNumber pageRang
static void form_and_insert_tuple(BrinBuildState *state);
static void union_tuples(BrinDesc *bdesc, BrinMemTuple *a,
BrinTuple *b);
-static void brin_vacuum_scan(Relation idxrel, BufferAccessStrategy strategy);
+static void brin_vacuum_scan(Relation idxrel, BufferAccessStrategy strategy, bool report_progress);
static bool add_values_to_range(Relation idxRel, BrinDesc *bdesc,
BrinMemTuple *dtup, Datum *values, bool *nulls);
static bool check_null_keys(BrinValues *bval, ScanKey *nullkeys, int nnullkeys);
@@ -953,7 +955,7 @@ brinvacuumcleanup(IndexVacuumInfo *info, IndexBulkDeleteResult *stats)
heapRel = table_open(IndexGetRelation(RelationGetRelid(info->index), false),
AccessShareLock);
- brin_vacuum_scan(info->index, info->strategy);
+ brin_vacuum_scan(info->index, info->strategy, info->report_progress);
brinsummarize(info->index, heapRel, BRIN_ALL_BLOCKRANGES, false,
&stats->num_index_tuples, &stats->num_index_tuples);
@@ -1635,16 +1637,24 @@ union_tuples(BrinDesc *bdesc, BrinMemTuple *a, BrinTuple *b)
* and such.
*/
static void
-brin_vacuum_scan(Relation idxrel, BufferAccessStrategy strategy)
+brin_vacuum_scan(Relation idxrel, BufferAccessStrategy strategy, bool report_progress)
{
BlockNumber nblocks;
BlockNumber blkno;
+ const int initprog_index[] = {
+ PROGRESS_SCAN_BLOCKS_DONE,
+ PROGRESS_SCAN_BLOCKS_TOTAL
+ };
+ int64 initprog_val[2];
/*
* Scan the index in physical order, and clean up any possible mess in
* each page.
*/
nblocks = RelationGetNumberOfBlocks(idxrel);
+ if (report_progress)
+ pgstat_progress_update_param(PROGRESS_SCAN_BLOCKS_TOTAL,
+ nblocks);
for (blkno = 0; blkno < nblocks; blkno++)
{
Buffer buf;
@@ -1656,9 +1666,20 @@ brin_vacuum_scan(Relation idxrel, BufferAccessStrategy strategy)
brin_page_cleanup(idxrel, buf);
+ if (report_progress)
+ pgstat_progress_update_param(PROGRESS_SCAN_BLOCKS_DONE,
+ blkno + 1);
+
ReleaseBuffer(buf);
}
+ if (report_progress)
+ {
+ initprog_val[0] = 0;
+ initprog_val[1] = 0;
+ pgstat_progress_update_multi_param(2, initprog_index, initprog_val);
+ }
+
/*
* Update all upper pages in the index's FSM, as well. This ensures not
* only that we propagate leaf-page FSM updates made by brin_page_cleanup,
diff --git a/src/backend/access/gin/ginvacuum.c b/src/backend/access/gin/ginvacuum.c
index a276eb020b5..714586040aa 100644
--- a/src/backend/access/gin/ginvacuum.c
+++ b/src/backend/access/gin/ginvacuum.c
@@ -24,6 +24,8 @@
#include "storage/lmgr.h"
#include "storage/predicate.h"
#include "utils/memutils.h"
+#include "commands/progress.h"
+#include "pgstat.h"
struct GinVacuumState
{
@@ -571,6 +573,14 @@ ginbulkdelete(IndexVacuumInfo *info, IndexBulkDeleteResult *stats,
Buffer buffer;
BlockNumber rootOfPostingTree[BLCKSZ / (sizeof(IndexTupleData) + sizeof(ItemId))];
uint32 nRoot;
+ BlockNumber num_pages;
+ bool needLock;
+ int blocks_scanned = 0;
+ const int initprog_index[] = {
+ PROGRESS_SCAN_BLOCKS_DONE,
+ PROGRESS_SCAN_BLOCKS_TOTAL
+ };
+ int64 initprog_val[2];
gvs.tmpCxt = AllocSetContextCreate(CurrentMemoryContext,
"Gin vacuum temporary context",
@@ -635,6 +645,19 @@ ginbulkdelete(IndexVacuumInfo *info, IndexBulkDeleteResult *stats,
RBM_NORMAL, info->strategy);
}
+ needLock = !RELATION_IS_LOCAL(index);
+
+ /* Get the current relation length */
+ if (needLock)
+ LockRelationForExtension(index, ExclusiveLock);
+ num_pages = RelationGetNumberOfBlocks(index);
+ if (needLock)
+ UnlockRelationForExtension(index, ExclusiveLock);
+
+ if (info->report_progress)
+ pgstat_progress_update_param(PROGRESS_SCAN_BLOCKS_TOTAL,
+ num_pages);
+
/* right now we found leftmost page in entry's BTree */
for (;;)
@@ -676,9 +699,20 @@ ginbulkdelete(IndexVacuumInfo *info, IndexBulkDeleteResult *stats,
buffer = ReadBufferExtended(index, MAIN_FORKNUM, blkno,
RBM_NORMAL, info->strategy);
+ blocks_scanned++;
+ if (info->report_progress)
+ pgstat_progress_update_param(PROGRESS_SCAN_BLOCKS_DONE,
+ blocks_scanned + 1);
LockBuffer(buffer, GIN_EXCLUSIVE);
}
+ if (info->report_progress)
+ {
+ initprog_val[0] = 0;
+ initprog_val[1] = 0;
+ pgstat_progress_update_multi_param(2, initprog_index, initprog_val);
+ }
+
MemoryContextDelete(gvs.tmpCxt);
return gvs.result;
@@ -694,6 +728,12 @@ ginvacuumcleanup(IndexVacuumInfo *info, IndexBulkDeleteResult *stats)
BlockNumber totFreePages;
GinState ginstate;
GinStatsData idxStat;
+ int blocks_scanned = 0;
+ const int initprog_index[] = {
+ PROGRESS_SCAN_BLOCKS_DONE,
+ PROGRESS_SCAN_BLOCKS_TOTAL
+ };
+ int64 initprog_val[2];
/*
* In an autovacuum analyze, we want to clean up pending insertions.
@@ -744,6 +784,9 @@ ginvacuumcleanup(IndexVacuumInfo *info, IndexBulkDeleteResult *stats)
totFreePages = 0;
+ if (info->report_progress)
+ pgstat_progress_update_param(PROGRESS_SCAN_BLOCKS_TOTAL,
+ npages);
for (blkno = GIN_ROOT_BLKNO; blkno < npages; blkno++)
{
Buffer buffer;
@@ -774,9 +817,21 @@ ginvacuumcleanup(IndexVacuumInfo *info, IndexBulkDeleteResult *stats)
idxStat.nEntries += PageGetMaxOffsetNumber(page);
}
+ blocks_scanned++;
+ if (info->report_progress)
+ pgstat_progress_update_param(PROGRESS_SCAN_BLOCKS_DONE,
+ blocks_scanned + 1);
+
UnlockReleaseBuffer(buffer);
}
+ if (info->report_progress)
+ {
+ initprog_val[0] = 0;
+ initprog_val[1] = 0;
+ pgstat_progress_update_multi_param(2, initprog_index, initprog_val);
+ }
+
/* Update the metapage with accurate page and entry counts */
idxStat.nTotalPages = npages;
ginUpdateStats(info->index, &idxStat, false);
diff --git a/src/backend/access/gist/gistvacuum.c b/src/backend/access/gist/gistvacuum.c
index 0663193531a..e7d13c9eb6e 100644
--- a/src/backend/access/gist/gistvacuum.c
+++ b/src/backend/access/gist/gistvacuum.c
@@ -23,6 +23,8 @@
#include "storage/indexfsm.h"
#include "storage/lmgr.h"
#include "utils/memutils.h"
+#include "commands/progress.h"
+#include "pgstat.h"
/* Working state needed by gistbulkdelete */
typedef struct
@@ -131,6 +133,11 @@ gistvacuumscan(IndexVacuumInfo *info, IndexBulkDeleteResult *stats,
bool needLock;
BlockNumber blkno;
MemoryContext oldctx;
+ const int initprog_index[] = {
+ PROGRESS_SCAN_BLOCKS_DONE,
+ PROGRESS_SCAN_BLOCKS_TOTAL
+ };
+ int64 initprog_val[2];
/*
* Reset fields that track information about the entire index now. This
@@ -215,9 +222,26 @@ gistvacuumscan(IndexVacuumInfo *info, IndexBulkDeleteResult *stats,
/* Quit if we've scanned the whole relation */
if (blkno >= num_pages)
break;
+
+ if (info->report_progress)
+ pgstat_progress_update_param(PROGRESS_SCAN_BLOCKS_TOTAL,
+ num_pages);
+
/* Iterate over pages, then loop back to recheck length */
for (; blkno < num_pages; blkno++)
+ {
gistvacuumpage(&vstate, blkno, blkno);
+ if (info->report_progress)
+ pgstat_progress_update_param(PROGRESS_SCAN_BLOCKS_DONE,
+ blkno + 1);
+ }
+ }
+
+ if (info->report_progress)
+ {
+ initprog_val[0] = 0;
+ initprog_val[1] = 0;
+ pgstat_progress_update_multi_param(2, initprog_index, initprog_val);
}
/*
diff --git a/src/backend/access/hash/hash.c b/src/backend/access/hash/hash.c
index 81c7da7ec69..2b4eed6aae1 100644
--- a/src/backend/access/hash/hash.c
+++ b/src/backend/access/hash/hash.c
@@ -31,6 +31,7 @@
#include "utils/builtins.h"
#include "utils/index_selfuncs.h"
#include "utils/rel.h"
+#include "storage/lmgr.h"
/* Working state for hashbuild and its callback */
typedef struct
@@ -469,9 +470,21 @@ hashbulkdelete(IndexVacuumInfo *info, IndexBulkDeleteResult *stats,
Buffer metabuf = InvalidBuffer;
HashMetaPage metap;
HashMetaPage cachedmetap;
+ int blocks_scanned;
+ int bucket_blocks_scanned;
+ BlockNumber num_pages;
+ bool needLock;
+ const int initprog_index[] = {
+ PROGRESS_SCAN_BLOCKS_DONE,
+ PROGRESS_SCAN_BLOCKS_TOTAL
+ };
+ int64 initprog_val[2];
+
tuples_removed = 0;
num_index_tuples = 0;
+ blocks_scanned = 0;
+ bucket_blocks_scanned = 0;
/*
* We need a copy of the metapage so that we can use its hashm_spares[]
@@ -489,6 +502,19 @@ hashbulkdelete(IndexVacuumInfo *info, IndexBulkDeleteResult *stats,
cur_bucket = 0;
cur_maxbucket = orig_maxbucket;
+ needLock = !RELATION_IS_LOCAL(rel);
+
+ /* Get the current relation length */
+ if (needLock)
+ LockRelationForExtension(rel, ExclusiveLock);
+ num_pages = RelationGetNumberOfBlocks(rel);
+ if (needLock)
+ UnlockRelationForExtension(rel, ExclusiveLock);
+
+ if (info->report_progress)
+ pgstat_progress_update_param(PROGRESS_SCAN_BLOCKS_TOTAL,
+ num_pages);
+
loop_top:
while (cur_bucket <= cur_maxbucket)
{
@@ -504,6 +530,7 @@ loop_top:
bucket_blkno = BUCKET_TO_BLKNO(cachedmetap, cur_bucket);
blkno = bucket_blkno;
+ blocks_scanned++;
/*
* We need to acquire a cleanup lock on the primary bucket page to out
@@ -550,10 +577,14 @@ loop_top:
cachedmetap->hashm_highmask,
cachedmetap->hashm_lowmask, &tuples_removed,
&num_index_tuples, split_cleanup,
- callback, callback_state);
+ callback, callback_state, info->report_progress,
+ &bucket_blocks_scanned);
_hash_dropbuf(rel, bucket_buf);
+ pgstat_progress_update_param(PROGRESS_SCAN_BLOCKS_DONE,
+ blocks_scanned + bucket_blocks_scanned);
+
/* Advance to next bucket */
cur_bucket++;
}
@@ -633,6 +664,13 @@ loop_top:
stats->tuples_removed += tuples_removed;
/* hashvacuumcleanup will fill in num_pages */
+ if (info->report_progress)
+ {
+ initprog_val[0] = 0;
+ initprog_val[1] = 0;
+ pgstat_progress_update_multi_param(2, initprog_index, initprog_val);
+ }
+
return stats;
}
@@ -686,7 +724,8 @@ hashbucketcleanup(Relation rel, Bucket cur_bucket, Buffer bucket_buf,
uint32 maxbucket, uint32 highmask, uint32 lowmask,
double *tuples_removed, double *num_index_tuples,
bool split_cleanup,
- IndexBulkDeleteCallback callback, void *callback_state)
+ IndexBulkDeleteCallback callback, void *callback_state,
+ bool report_progress, int *bucket_blocks_scanned)
{
BlockNumber blkno;
Buffer buf;
@@ -718,6 +757,8 @@ hashbucketcleanup(Relation rel, Bucket cur_bucket, Buffer bucket_buf,
page = BufferGetPage(buf);
opaque = (HashPageOpaque) PageGetSpecialPointer(page);
+ bucket_blocks_scanned++;
+
/* Scan each tuple in page */
maxoffno = PageGetMaxOffsetNumber(page);
for (offno = FirstOffsetNumber;
@@ -916,4 +957,5 @@ hashbucketcleanup(Relation rel, Bucket cur_bucket, Buffer bucket_buf,
bstrategy);
else
LockBuffer(bucket_buf, BUFFER_LOCK_UNLOCK);
+
}
diff --git a/src/backend/access/hash/hashpage.c b/src/backend/access/hash/hashpage.c
index 159646c7c3e..538d153df7c 100644
--- a/src/backend/access/hash/hashpage.c
+++ b/src/backend/access/hash/hashpage.c
@@ -758,7 +758,7 @@ restart_expand:
hashbucketcleanup(rel, old_bucket, buf_oblkno, start_oblkno, NULL,
maxbucket, highmask, lowmask, NULL, NULL, true,
- NULL, NULL);
+ NULL, NULL, NULL, NULL);
_hash_dropbuf(rel, buf_oblkno);
@@ -1326,7 +1326,7 @@ _hash_splitbucket(Relation rel,
hashbucketcleanup(rel, obucket, bucket_obuf,
BufferGetBlockNumber(bucket_obuf), NULL,
maxbucket, highmask, lowmask, NULL, NULL, true,
- NULL, NULL);
+ NULL, NULL, NULL, NULL);
}
else
{
diff --git a/src/backend/access/heap/vacuumlazy.c b/src/backend/access/heap/vacuumlazy.c
index cd603e6aa41..6900b9ff5db 100644
--- a/src/backend/access/heap/vacuumlazy.c
+++ b/src/backend/access/heap/vacuumlazy.c
@@ -344,6 +344,9 @@ heap_vacuum_rel(Relation rel, VacuumParams *params,
pgstat_progress_start_command(PROGRESS_COMMAND_VACUUM,
RelationGetRelid(rel));
+ pgstat_progress_update_param(PROGRESS_VACUUM_LEADER_PID,
+ MyProcPid);
+
vacuum_set_xid_limits(rel,
params->freeze_min_age,
params->freeze_table_age,
@@ -2508,10 +2511,15 @@ lazy_vacuum_one_index(Relation indrel, IndexBulkDeleteResult *istat,
{
IndexVacuumInfo ivinfo;
LVSavedErrInfo saved_err_info;
+ const int initprog_index[] = {
+ PROGRESS_VACUUM_PHASE,
+ PROGRESS_VACUUM_CURRENT_INDRELID
+ };
+ int64 initprog_val[2];
ivinfo.index = indrel;
ivinfo.analyze_only = false;
- ivinfo.report_progress = false;
+ ivinfo.report_progress = true;
ivinfo.estimated_count = true;
ivinfo.message_level = elevel;
ivinfo.num_heap_tuples = reltuples;
@@ -2529,9 +2537,18 @@ lazy_vacuum_one_index(Relation indrel, IndexBulkDeleteResult *istat,
VACUUM_ERRCB_PHASE_VACUUM_INDEX,
InvalidBlockNumber, InvalidOffsetNumber);
+ /* Report that we're vacuuming the index, advertising the indrelid */
+ initprog_val[0] = PROGRESS_VACUUM_PHASE_VACUUM_INDEX;
+ initprog_val[1] = RelationGetRelid(indrel);
+ pgstat_progress_update_multi_param(2, initprog_index, initprog_val);
+
/* Do bulk deletion */
istat = vac_bulkdel_one_index(&ivinfo, istat, (void *) vacrel->dead_items);
+ /* Report that we're done vacuuming the index */
+ pgstat_progress_update_param(PROGRESS_VACUUM_CURRENT_INDRELID,
+ 0);
+
/* Revert to the previous phase information for error traceback */
restore_vacuum_error_info(vacrel, &saved_err_info);
pfree(vacrel->indname);
@@ -2556,10 +2573,15 @@ lazy_cleanup_one_index(Relation indrel, IndexBulkDeleteResult *istat,
{
IndexVacuumInfo ivinfo;
LVSavedErrInfo saved_err_info;
+ const int initprog_index[] = {
+ PROGRESS_VACUUM_PHASE,
+ PROGRESS_VACUUM_CURRENT_INDRELID
+ };
+ int64 initprog_val[2];
ivinfo.index = indrel;
ivinfo.analyze_only = false;
- ivinfo.report_progress = false;
+ ivinfo.report_progress = true;
ivinfo.estimated_count = estimated_count;
ivinfo.message_level = elevel;
@@ -2578,8 +2600,18 @@ lazy_cleanup_one_index(Relation indrel, IndexBulkDeleteResult *istat,
VACUUM_ERRCB_PHASE_INDEX_CLEANUP,
InvalidBlockNumber, InvalidOffsetNumber);
+ /* Report that we're cleaning the index, advertising the indrelid */
+ initprog_val[0] = PROGRESS_VACUUM_PHASE_INDEX_CLEANUP;
+ initprog_val[1] = RelationGetRelid(indrel);
+ pgstat_progress_update_multi_param(2, initprog_index, initprog_val);
+
istat = vac_cleanup_one_index(&ivinfo, istat);
+ /* Report that we're done cleaning the index */
+ initprog_val[0] = 0;
+ initprog_val[1] = 0;
+ pgstat_progress_update_multi_param(2, initprog_index, initprog_val);
+
/* Revert to the previous phase information for error traceback */
restore_vacuum_error_info(vacrel, &saved_err_info);
pfree(vacrel->indname);
diff --git a/src/backend/access/nbtree/nbtree.c b/src/backend/access/nbtree/nbtree.c
index dfce06dc49f..15c11fcab81 100644
--- a/src/backend/access/nbtree/nbtree.c
+++ b/src/backend/access/nbtree/nbtree.c
@@ -909,6 +909,11 @@ btvacuumscan(IndexVacuumInfo *info, IndexBulkDeleteResult *stats,
BlockNumber num_pages;
BlockNumber scanblkno;
bool needLock;
+ const int initprog_index[] = {
+ PROGRESS_SCAN_BLOCKS_DONE,
+ PROGRESS_SCAN_BLOCKS_TOTAL
+ };
+ int64 initprog_val[2];
/*
* Reset fields that track information about the entire index now. This
@@ -997,10 +1002,17 @@ btvacuumscan(IndexVacuumInfo *info, IndexBulkDeleteResult *stats,
btvacuumpage(&vstate, scanblkno);
if (info->report_progress)
pgstat_progress_update_param(PROGRESS_SCAN_BLOCKS_DONE,
- scanblkno);
+ scanblkno + 1);
}
}
+ if (info->report_progress)
+ {
+ initprog_val[0] = 0;
+ initprog_val[1] = 0;
+ pgstat_progress_update_multi_param(2, initprog_index, initprog_val);
+ }
+
/* Set statistics num_pages field to final size of index */
stats->num_pages = num_pages;
diff --git a/src/backend/access/spgist/spgvacuum.c b/src/backend/access/spgist/spgvacuum.c
index 76fb0374c42..c71345fcebe 100644
--- a/src/backend/access/spgist/spgvacuum.c
+++ b/src/backend/access/spgist/spgvacuum.c
@@ -27,6 +27,8 @@
#include "storage/indexfsm.h"
#include "storage/lmgr.h"
#include "utils/snapmgr.h"
+#include "commands/progress.h"
+#include "pgstat.h"
/* Entry in pending-list of TIDs we need to revisit */
@@ -797,6 +799,12 @@ spgvacuumscan(spgBulkDeleteState *bds)
bool needLock;
BlockNumber num_pages,
blkno;
+ int blocks_scanned = 0;
+ const int initprog_index[] = {
+ PROGRESS_SCAN_BLOCKS_DONE,
+ PROGRESS_SCAN_BLOCKS_TOTAL
+ };
+ int64 initprog_val[2];
/* Finish setting up spgBulkDeleteState */
initSpGistState(&bds->spgstate, index);
@@ -836,6 +844,11 @@ spgvacuumscan(spgBulkDeleteState *bds)
/* Quit if we've scanned the whole relation */
if (blkno >= num_pages)
break;
+
+ if (bds->info->report_progress)
+ pgstat_progress_update_param(PROGRESS_SCAN_BLOCKS_TOTAL,
+ num_pages);
+
/* Iterate over pages, then loop back to recheck length */
for (; blkno < num_pages; blkno++)
{
@@ -843,9 +856,21 @@ spgvacuumscan(spgBulkDeleteState *bds)
/* empty the pending-list after each page */
if (bds->pendingList != NULL)
spgprocesspending(bds);
+
+ blocks_scanned++;
+ if (bds->info->report_progress)
+ pgstat_progress_update_param(PROGRESS_SCAN_BLOCKS_DONE,
+ blocks_scanned + 1);
}
}
+ if (bds->info->report_progress)
+ {
+ initprog_val[0] = 0;
+ initprog_val[1] = 0;
+ pgstat_progress_update_multi_param(2, initprog_index, initprog_val);
+ }
+
/* Propagate local lastUsedPages cache to metablock */
SpGistUpdateMetaPage(index);
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index 61b515cdb85..9e0dc39314c 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1128,6 +1128,17 @@ CREATE VIEW pg_stat_progress_vacuum AS
FROM pg_stat_get_progress_info('VACUUM') AS S
LEFT JOIN pg_database D ON S.datid = D.oid;
+CREATE VIEW pg_stat_progress_vacuum_worker AS
+ SELECT
+ S.pid,
+ S.param9 leader_pid,
+ S.param8 AS indrelid,
+ S.param16 index_blks_total,
+ S.param17 AS index_blks_scanned
+ FROM pg_stat_get_progress_info('VACUUM') AS S
+ LEFT JOIN pg_database D ON S.datid = D.oid
+ WHERE S.param8 > 0;
+
CREATE VIEW pg_stat_progress_cluster AS
SELECT
S.pid AS pid,
diff --git a/src/backend/commands/vacuumparallel.c b/src/backend/commands/vacuumparallel.c
index 0d61c8ec74a..aed186c0c0c 100644
--- a/src/backend/commands/vacuumparallel.c
+++ b/src/backend/commands/vacuumparallel.c
@@ -29,6 +29,7 @@
#include "access/amapi.h"
#include "access/table.h"
#include "catalog/index.h"
+#include "commands/progress.h"
#include "commands/vacuum.h"
#include "optimizer/paths.h"
#include "pgstat.h"
@@ -942,6 +943,7 @@ parallel_vacuum_main(dsm_segment *seg, shm_toc *toc)
int nindexes;
char *sharedquery;
ErrorContextCallback errcallback;
+ PGPROC *leader = MyProc->lockGroupLeader;
/*
* A parallel vacuum worker must have only PROC_IN_VACUUM flag since we
@@ -965,6 +967,14 @@ parallel_vacuum_main(dsm_segment *seg, shm_toc *toc)
*/
rel = table_open(shared->relid, ShareUpdateExclusiveLock);
+ /*
+ * Track progress of current index being vacuumed
+ */
+ pgstat_progress_start_command(PROGRESS_COMMAND_VACUUM,
+ RelationGetRelid(rel));
+
+ pgstat_progress_update_param(PROGRESS_VACUUM_LEADER_PID, leader->pid);
+
/*
* Open all indexes. indrels are sorted in order by OID, which should be
* matched to the leader's one.
@@ -1035,6 +1045,7 @@ parallel_vacuum_main(dsm_segment *seg, shm_toc *toc)
vac_close_indexes(nindexes, indrels, RowExclusiveLock);
table_close(rel, ShareUpdateExclusiveLock);
FreeAccessStrategy(pvs.bstrategy);
+ pgstat_progress_end_command();
}
/*
diff --git a/src/include/access/hash.h b/src/include/access/hash.h
index 1cce865be2b..c00bb76e3e2 100644
--- a/src/include/access/hash.h
+++ b/src/include/access/hash.h
@@ -478,6 +478,7 @@ extern void hashbucketcleanup(Relation rel, Bucket cur_bucket,
uint32 maxbucket, uint32 highmask, uint32 lowmask,
double *tuples_removed, double *num_index_tuples,
bool split_cleanup,
- IndexBulkDeleteCallback callback, void *callback_state);
+ IndexBulkDeleteCallback callback, void *callback_state,
+ bool report_progress, int *bucket_blocks_scanned);
#endif /* HASH_H */
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index d7bf16368bd..4387a7c1f16 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -25,6 +25,8 @@
#define PROGRESS_VACUUM_NUM_INDEX_VACUUMS 4
#define PROGRESS_VACUUM_MAX_DEAD_TUPLES 5
#define PROGRESS_VACUUM_NUM_DEAD_TUPLES 6
+#define PROGRESS_VACUUM_CURRENT_INDRELID 7
+#define PROGRESS_VACUUM_LEADER_PID 8
/* Phases of vacuum (as advertised via PROGRESS_VACUUM_PHASE) */
#define PROGRESS_VACUUM_PHASE_SCAN_HEAP 1
--
2.17.1
--OwLcNYc0lM97+oe1--
^ permalink raw reply [nested|flat] 44+ messages in thread
* Re: Add index scan progress to pg_stat_progress_vacuum
@ 2021-12-20 18:37 Peter Geoghegan <[email protected]>
parent: Bossart, Nathan <[email protected]>
1 sibling, 1 reply; 44+ messages in thread
From: Peter Geoghegan @ 2021-12-20 18:37 UTC (permalink / raw)
To: Bossart, Nathan <[email protected]>; +Cc: Imseih (AWS), Sami <[email protected]>; pgsql-hackers
On Wed, Dec 15, 2021 at 2:10 PM Bossart, Nathan <[email protected]> wrote:
> nitpick: Shouldn't index_blks_scanned be index_blks_vacuumed? IMO it
> is more analogous to heap_blks_vacuumed.
+1.
> This will tell us which indexes are currently being vacuumed and the
> current progress of those operations, but it doesn't tell us which
> indexes have already been vacuumed or which ones are pending vacuum.
VACUUM will process a table's indexes in pg_class OID order (outside
of parallel VACUUM, I suppose). See comments about sort order above
RelationGetIndexList().
Anyway, it might be useful to add ordinal numbers to each index, that
line up with this processing/OID order. It would also be reasonable to
display the same number in log_autovacuum* (and VACUUM VERBOSE)
per-index output, to reinforce the idea. Note that we don't
necessarily display a distinct line for each distinct index in this
log output, which is why including the ordinal number there makes
sense.
> I wish option #1 was cleaner, because I think it would be really nice
> to have all this information in a single row.
I do too. I agree with the specific points you raise in your remarks
about what you've called options #2 and #3, but those options still
seem unappealing to me.
--
Peter Geoghegan
^ permalink raw reply [nested|flat] 44+ messages in thread
* Re: Add index scan progress to pg_stat_progress_vacuum
@ 2021-12-20 19:05 Peter Geoghegan <[email protected]>
parent: Imseih (AWS), Sami <[email protected]>
2 siblings, 1 reply; 44+ messages in thread
From: Peter Geoghegan @ 2021-12-20 19:05 UTC (permalink / raw)
To: Imseih (AWS), Sami <[email protected]>; +Cc: pgsql-hackers
On Wed, Dec 1, 2021 at 2:59 PM Imseih (AWS), Sami <[email protected]> wrote:
> The current implementation of pg_stat_progress_vacuum does not provide progress on which index is being vacuumed making it difficult for a user to determine if the "vacuuming indexes" phase is making progress.
I notice that your patch largely assumes that indexes can be treated
like heap relations, in the sense that they're scanned sequentially,
and process each block exactly once (or exactly once per "pass"). But
that isn't quite true. There are a few differences that seem like they
might matter:
* An ambulkdelete() scan of an index cannot take the size of the
relation once, at the start, and ignore any blocks that are added
after the scan begins. And so the code may need to re-establish the
total size of the index multiple times, to make sure no index tuples
are missed -- there may be index tuples that VACUUM needs to process
that appear in later pages due to concurrent page splits. You don't
have the issue with things like IndexBulkDeleteResult.num_pages,
because they report on the index after ambulkdelete/amvacuumcleanup
return (they're not granular progress indicators).
* Some index AMs don't work like nbtree and GiST in that they cannot
do their scan sequentially -- they have to do something like a
logical/keyspace order scan instead, which is *totally* different to
heapam (not just a bit different). There is no telling how many times
each page will be accessed in these other index AMs, and in what
order, even under optimal conditions. We should arguably not even try
to provide any granular progress information here, since it'll
probably be too messy.
I'm not sure what to recommend for your patch, in light of this. Maybe
you should change the names of the new columns to own the squishiness.
For example, instead of using the name index_blks_total, you might
instead use the name index_blks_initial. That might be enough to avoid
user confusion when we scan more blocks than the index initially
contained (within a single ambulkdelete scan).
Note also that we have to do something called backtracking in
btvacuumpage(), which you've ignored -- that's another reasonably
common way that we'll end up scanning a page twice. But that probably
should just be ignored -- it's too narrow a case to be worth caring
about.
--
Peter Geoghegan
^ permalink raw reply [nested|flat] 44+ messages in thread
* Re: Add index scan progress to pg_stat_progress_vacuum
@ 2021-12-20 19:27 Justin Pryzby <[email protected]>
parent: Imseih (AWS), Sami <[email protected]>
2 siblings, 1 reply; 44+ messages in thread
From: Justin Pryzby @ 2021-12-20 19:27 UTC (permalink / raw)
To: Imseih (AWS), Sami <[email protected]>; +Cc: pgsql-hackers
This view also doesn't show vacuum progress across a partitioned table.
For comparison:
pg_stat_progress_create_index (added in v12) has:
partitions_total
partitions_done
pg_stat_progress_analyze (added in v13) has:
child_tables_total
child_tables_done
pg_stat_progress_cluster should have something similar.
--
Justin Pryzby
System Administrator
Telsasoft
+1-952-707-8581
^ permalink raw reply [nested|flat] 44+ messages in thread
* Re: Add index scan progress to pg_stat_progress_vacuum
@ 2021-12-23 08:44 Masahiko Sawada <[email protected]>
parent: Peter Geoghegan <[email protected]>
0 siblings, 1 reply; 44+ messages in thread
From: Masahiko Sawada @ 2021-12-23 08:44 UTC (permalink / raw)
To: Peter Geoghegan <[email protected]>; +Cc: Bossart, Nathan <[email protected]>; Imseih (AWS), Sami <[email protected]>; pgsql-hackers
On Tue, Dec 21, 2021 at 3:37 AM Peter Geoghegan <[email protected]> wrote:
>
> On Wed, Dec 15, 2021 at 2:10 PM Bossart, Nathan <[email protected]> wrote:
> > nitpick: Shouldn't index_blks_scanned be index_blks_vacuumed? IMO it
> > is more analogous to heap_blks_vacuumed.
>
> +1.
>
> > This will tell us which indexes are currently being vacuumed and the
> > current progress of those operations, but it doesn't tell us which
> > indexes have already been vacuumed or which ones are pending vacuum.
>
> VACUUM will process a table's indexes in pg_class OID order (outside
> of parallel VACUUM, I suppose). See comments about sort order above
> RelationGetIndexList().
Right.
>
> Anyway, it might be useful to add ordinal numbers to each index, that
> line up with this processing/OID order. It would also be reasonable to
> display the same number in log_autovacuum* (and VACUUM VERBOSE)
> per-index output, to reinforce the idea. Note that we don't
> necessarily display a distinct line for each distinct index in this
> log output, which is why including the ordinal number there makes
> sense.
An alternative idea would be to show the number of indexes on the
table and the number of indexes that have been processed in the
leader's entry of pg_stat_progress_vacuum. Even in parallel vacuum
cases, since we have index vacuum status for each index it would not
be hard for the leader process to count how many indexes have been
processed.
Regarding the details of the progress of index vacuum, I'm not sure
this progress information can fit for pg_stat_progress_vacuum. As
Peter already mentioned, the behavior quite varies depending on index
AM.
Regards,
--
Masahiko Sawada
EDB: https://www.enterprisedb.com/
^ permalink raw reply [nested|flat] 44+ messages in thread
* Re: Add index scan progress to pg_stat_progress_vacuum
@ 2021-12-23 10:49 Andrey Lepikhov <[email protected]>
parent: Peter Geoghegan <[email protected]>
0 siblings, 0 replies; 44+ messages in thread
From: Andrey Lepikhov @ 2021-12-23 10:49 UTC (permalink / raw)
To: Peter Geoghegan <[email protected]>; Imseih (AWS), Sami <[email protected]>; +Cc: pgsql-hackers
On 21/12/2021 00:05, Peter Geoghegan wrote:
> * Some index AMs don't work like nbtree and GiST in that they cannot
> do their scan sequentially -- they have to do something like a
> logical/keyspace order scan instead, which is *totally* different to
> heapam (not just a bit different). There is no telling how many times
> each page will be accessed in these other index AMs, and in what
> order, even under optimal conditions. We should arguably not even try
> to provide any granular progress information here, since it'll
> probably be too messy.
Maybe we could add callbacks into AM interface for
send/receive/representation implementation of progress?
So AM would define a set of parameters to send into stat collector and
show to users.
--
regards,
Andrey Lepikhov
Postgres Professional
^ permalink raw reply [nested|flat] 44+ messages in thread
* Re: Add index scan progress to pg_stat_progress_vacuum
@ 2021-12-27 17:59 Justin Pryzby <[email protected]>
parent: Justin Pryzby <[email protected]>
0 siblings, 0 replies; 44+ messages in thread
From: Justin Pryzby @ 2021-12-27 17:59 UTC (permalink / raw)
To: Imseih (AWS), Sami <[email protected]>; +Cc: pgsql-hackers
Please send your patches as *.diff or *.patch, so they're processed by the
patch tester. Preferably with commit messages; git format-patch is the usual
tool for this.
http://cfbot.cputube.org/sami-imseih.html
(Occasionally, it's also useful to send a *.txt to avoid the cfbot processing
the wrong thing, in case one sends an unrelated, secondary patch, or sends
fixes to a patch as a "relative patch" which doesn't include the main patch.)
I'm including a patch rebased on 8e1fae193.
^ permalink raw reply [nested|flat] 44+ messages in thread
* Re: Add index scan progress to pg_stat_progress_vacuum
@ 2021-12-28 00:13 Imseih (AWS), Sami <[email protected]>
parent: Masahiko Sawada <[email protected]>
0 siblings, 1 reply; 44+ messages in thread
From: Imseih (AWS), Sami @ 2021-12-28 00:13 UTC (permalink / raw)
To: Masahiko Sawada <[email protected]>; Peter Geoghegan <[email protected]>; +Cc: Bossart, Nathan <[email protected]>; pgsql-hackers
I do agree that tracking progress by # of blocks scanned is not deterministic for all index types.
Based on this feedback, I went back to the drawing board on this.
Something like below may make more sense.
In pg_stat_progress_vacuum, introduce 2 new columns:
1. total_index_vacuum - total # of indexes to vacuum
2. max_cycle_time - the time in seconds of the longest index cycle.
Introduce another view called pg_stat_progress_vacuum_index_cycle:
postgres=# \d pg_stat_progress_vacuum_index_cycle
View "public.pg_stat_progress_vacuum_worker"
Column | Type | Collation | Nullable | Default
----------------+---------+-----------+----------+---------
pid | integer | | | <<<-- the PID of the vacuum worker ( or leader if it's doing index vacuuming )
leader_pid | bigint | | | <<<-- the leader PID to allow this view to be joined back to pg_stat_progress_vacuum
indrelid | bigint | | | <<<- the index relid of the index being vacuumed
ordinal_position | bigint | | | <<<- the processing position, which will give an idea of the processing position of the index being vacuumed.
dead_tuples_removed | bigint | | <<<- the number of dead rows removed in the current cycle for the index.
Having this information, one can
1. Determine which index is being vacuumed. For monitoring tools, this can help identify the index that accounts for most of the index vacuuming time.
2. Having the processing order of the current index will allow the user to determine how many of the total indexes has been completed in the current cycle.
3. dead_tuples_removed will show progress on the index vacuum in the current cycle.
4. the max_cycle_time will give an idea on how long the longest index cycle took for the current vacuum operation.
On 12/23/21, 2:46 AM, "Masahiko Sawada" <[email protected]> wrote:
CAUTION: This email originated from outside of the organization. Do not click links or open attachments unless you can confirm the sender and know the content is safe.
On Tue, Dec 21, 2021 at 3:37 AM Peter Geoghegan <[email protected]> wrote:
>
> On Wed, Dec 15, 2021 at 2:10 PM Bossart, Nathan <[email protected]> wrote:
> > nitpick: Shouldn't index_blks_scanned be index_blks_vacuumed? IMO it
> > is more analogous to heap_blks_vacuumed.
>
> +1.
>
> > This will tell us which indexes are currently being vacuumed and the
> > current progress of those operations, but it doesn't tell us which
> > indexes have already been vacuumed or which ones are pending vacuum.
>
> VACUUM will process a table's indexes in pg_class OID order (outside
> of parallel VACUUM, I suppose). See comments about sort order above
> RelationGetIndexList().
Right.
>
> Anyway, it might be useful to add ordinal numbers to each index, that
> line up with this processing/OID order. It would also be reasonable to
> display the same number in log_autovacuum* (and VACUUM VERBOSE)
> per-index output, to reinforce the idea. Note that we don't
> necessarily display a distinct line for each distinct index in this
> log output, which is why including the ordinal number there makes
> sense.
An alternative idea would be to show the number of indexes on the
table and the number of indexes that have been processed in the
leader's entry of pg_stat_progress_vacuum. Even in parallel vacuum
cases, since we have index vacuum status for each index it would not
be hard for the leader process to count how many indexes have been
processed.
Regarding the details of the progress of index vacuum, I'm not sure
this progress information can fit for pg_stat_progress_vacuum. As
Peter already mentioned, the behavior quite varies depending on index
AM.
Regards,
--
Masahiko Sawada
EDB: https://www.enterprisedb.com/
^ permalink raw reply [nested|flat] 44+ messages in thread
* Re: Add index scan progress to pg_stat_progress_vacuum
@ 2021-12-29 16:44 Imseih (AWS), Sami <[email protected]>
parent: Imseih (AWS), Sami <[email protected]>
0 siblings, 1 reply; 44+ messages in thread
From: Imseih (AWS), Sami @ 2021-12-29 16:44 UTC (permalink / raw)
To: Masahiko Sawada <[email protected]>; Peter Geoghegan <[email protected]>; +Cc: Bossart, Nathan <[email protected]>; pgsql-hackers
Attached is the latest revision of the patch.
In "pg_stat_progress_vacuum", introduce 2 columns:
* total_index_vacuum : This is the # of indexes that will be vacuumed. Keep in mind that if failsafe mode kicks in mid-flight to the vacuum, Postgres may choose to forgo index scans. This value will be adjusted accordingly.
* max_index_vacuum_cycle_time : The total elapsed time for a index vacuum cycle is calculated and this value will be updated to reflect the longest vacuum cycle. Until the first cycle completes, this value will be 0. The purpose of this column is to give the user an idea of how long an index vacuum cycle takes to complete.
postgres=# \d pg_stat_progress_vacuum
View "pg_catalog.pg_stat_progress_vacuum"
Column | Type | Collation | Nullable | Default
-----------------------------+---------+-----------+----------+---------
pid | integer | | |
datid | oid | | |
datname | name | | |
relid | oid | | |
phase | text | | |
heap_blks_total | bigint | | |
heap_blks_scanned | bigint | | |
heap_blks_vacuumed | bigint | | |
index_vacuum_count | bigint | | |
max_dead_tuples | bigint | | |
num_dead_tuples | bigint | | |
total_index_vacuum | bigint | | |
max_index_vacuum_cycle_time | bigint | | |
Introduce a new view called "pg_stat_progress_vacuum_index". This view will track the progress of a worker ( or leader PID ) while it's vacuuming an index. It will expose some key columns:
* pid: The PID of the worker process
* leader_pid: The PID of the leader process. This is the column that can be joined with "pg_stat_progress_vacuum". leader_pid and pid can have the same value as a leader can also perform an index vacuum.
* indrelid: The relid of the index currently being vacuumed
* vacuum_cycle_ordinal_position: The processing position of the index being vacuumed. This can be useful to determine how many indexes out of the total indexes ( pg_stat_progress_vacuum.total_index_vacuum ) have been vacuumed
* index_tuples_vacuumed: This is the number of index tuples vacuumed for the index overall. This is useful to show that the vacuum is actually doing work, as the # of tuples keeps increasing.
postgres=# \d pg_stat_progress_vacuum_index
View "pg_catalog.pg_stat_progress_vacuum_index"
Column | Type | Collation | Nullable | Default
-------------------------------+---------+-----------+----------+---------
pid | integer | | |
leader_pid | bigint | | |
indrelid | bigint | | |
vacuum_cycle_ordinal_position | bigint | | |
index_tuples_vacuumed | bigint | | |
On 12/27/21, 6:12 PM, "Imseih (AWS), Sami" <[email protected]> wrote:
I do agree that tracking progress by # of blocks scanned is not deterministic for all index types.
Based on this feedback, I went back to the drawing board on this.
Something like below may make more sense.
In pg_stat_progress_vacuum, introduce 2 new columns:
1. total_index_vacuum - total # of indexes to vacuum
2. max_cycle_time - the time in seconds of the longest index cycle.
Introduce another view called pg_stat_progress_vacuum_index_cycle:
postgres=# \d pg_stat_progress_vacuum_index_cycle
View "public.pg_stat_progress_vacuum_worker"
Column | Type | Collation | Nullable | Default
----------------+---------+-----------+----------+---------
pid | integer | | | <<<-- the PID of the vacuum worker ( or leader if it's doing index vacuuming )
leader_pid | bigint | | | <<<-- the leader PID to allow this view to be joined back to pg_stat_progress_vacuum
indrelid | bigint | | | <<<- the index relid of the index being vacuumed
ordinal_position | bigint | | | <<<- the processing position, which will give an idea of the processing position of the index being vacuumed.
dead_tuples_removed | bigint | | <<<- the number of dead rows removed in the current cycle for the index.
Having this information, one can
1. Determine which index is being vacuumed. For monitoring tools, this can help identify the index that accounts for most of the index vacuuming time.
2. Having the processing order of the current index will allow the user to determine how many of the total indexes has been completed in the current cycle.
3. dead_tuples_removed will show progress on the index vacuum in the current cycle.
4. the max_cycle_time will give an idea on how long the longest index cycle took for the current vacuum operation.
On 12/23/21, 2:46 AM, "Masahiko Sawada" <[email protected]> wrote:
CAUTION: This email originated from outside of the organization. Do not click links or open attachments unless you can confirm the sender and know the content is safe.
On Tue, Dec 21, 2021 at 3:37 AM Peter Geoghegan <[email protected]> wrote:
>
> On Wed, Dec 15, 2021 at 2:10 PM Bossart, Nathan <[email protected]> wrote:
> > nitpick: Shouldn't index_blks_scanned be index_blks_vacuumed? IMO it
> > is more analogous to heap_blks_vacuumed.
>
> +1.
>
> > This will tell us which indexes are currently being vacuumed and the
> > current progress of those operations, but it doesn't tell us which
> > indexes have already been vacuumed or which ones are pending vacuum.
>
> VACUUM will process a table's indexes in pg_class OID order (outside
> of parallel VACUUM, I suppose). See comments about sort order above
> RelationGetIndexList().
Right.
>
> Anyway, it might be useful to add ordinal numbers to each index, that
> line up with this processing/OID order. It would also be reasonable to
> display the same number in log_autovacuum* (and VACUUM VERBOSE)
> per-index output, to reinforce the idea. Note that we don't
> necessarily display a distinct line for each distinct index in this
> log output, which is why including the ordinal number there makes
> sense.
An alternative idea would be to show the number of indexes on the
table and the number of indexes that have been processed in the
leader's entry of pg_stat_progress_vacuum. Even in parallel vacuum
cases, since we have index vacuum status for each index it would not
be hard for the leader process to count how many indexes have been
processed.
Regarding the details of the progress of index vacuum, I'm not sure
this progress information can fit for pg_stat_progress_vacuum. As
Peter already mentioned, the behavior quite varies depending on index
AM.
Regards,
--
Masahiko Sawada
EDB: https://www.enterprisedb.com/
Attachments:
[application/octet-stream] 0001-Add-index-scan-progress-to-pg_stat_progress_vacuum.patch (16.6K, ../../[email protected]/2-0001-Add-index-scan-progress-to-pg_stat_progress_vacuum.patch)
download | inline diff:
From 149631ce5c8c5cb5b05c92a7df1de3d8e431a3c2 Mon Sep 17 00:00:00 2001
From: "Imseih (AWS), Sami" <[email protected]>
Date: Tue, 28 Dec 2021 22:00:43 +0000
Subject: [PATCH 1/1] Add index scan progress to pg_stat_progress_vacuum.
Expose progress for the "vacuuming indexes" phase
of a VACUUM operation.
Author: Sami Imseih, based on suggestions by Nathan Bossart, Peter Geoghegan and Masahiko Sawada
Discussion: https://www.postgresql.org/message-id/5478DFCD-2333-401A-B2F0-0D186AB09228%40amazon.com
---
src/backend/access/gin/ginvacuum.c | 4 ++
src/backend/access/gist/gistvacuum.c | 4 ++
src/backend/access/hash/hash.c | 4 ++
src/backend/access/heap/vacuumlazy.c | 32 ++++++++++++++-
src/backend/access/nbtree/nbtree.c | 4 ++
src/backend/access/spgist/spgvacuum.c | 4 ++
src/backend/catalog/system_views.sql | 15 ++++++-
src/backend/commands/vacuumparallel.c | 22 +++++++++--
src/backend/utils/misc/pg_rusage.c | 56 ++++++++++++++++++++-------
src/include/commands/progress.h | 20 ++++++----
src/include/utils/pg_rusage.h | 1 +
11 files changed, 137 insertions(+), 29 deletions(-)
diff --git a/src/backend/access/gin/ginvacuum.c b/src/backend/access/gin/ginvacuum.c
index a276eb020b..07fd5271a1 100644
--- a/src/backend/access/gin/ginvacuum.c
+++ b/src/backend/access/gin/ginvacuum.c
@@ -24,6 +24,8 @@
#include "storage/lmgr.h"
#include "storage/predicate.h"
#include "utils/memutils.h"
+#include "commands/progress.h"
+#include "pgstat.h"
struct GinVacuumState
{
@@ -60,6 +62,8 @@ ginVacuumItemPointers(GinVacuumState *gvs, ItemPointerData *items,
if (gvs->callback(items + i, gvs->callback_state))
{
gvs->result->tuples_removed += 1;
+ pgstat_progress_update_param(PROGRESS_VACUUM_DEAD_TUPLES_VACUUMED, gvs->result->tuples_removed);
+
if (!tmpitems)
{
/*
diff --git a/src/backend/access/gist/gistvacuum.c b/src/backend/access/gist/gistvacuum.c
index 0663193531..ee77e5e405 100644
--- a/src/backend/access/gist/gistvacuum.c
+++ b/src/backend/access/gist/gistvacuum.c
@@ -23,6 +23,8 @@
#include "storage/indexfsm.h"
#include "storage/lmgr.h"
#include "utils/memutils.h"
+#include "commands/progress.h"
+#include "pgstat.h"
/* Working state needed by gistbulkdelete */
typedef struct
@@ -375,6 +377,8 @@ restart:
END_CRIT_SECTION();
vstate->stats->tuples_removed += ntodelete;
+ pgstat_progress_update_param(PROGRESS_VACUUM_DEAD_TUPLES_VACUUMED, vstate->stats->tuples_removed);
+
/* must recompute maxoff */
maxoff = PageGetMaxOffsetNumber(page);
}
diff --git a/src/backend/access/hash/hash.c b/src/backend/access/hash/hash.c
index 81c7da7ec6..0ac59fdb7e 100644
--- a/src/backend/access/hash/hash.c
+++ b/src/backend/access/hash/hash.c
@@ -31,6 +31,8 @@
#include "utils/builtins.h"
#include "utils/index_selfuncs.h"
#include "utils/rel.h"
+#include "commands/progress.h"
+#include "pgstat.h"
/* Working state for hashbuild and its callback */
typedef struct
@@ -631,6 +633,8 @@ loop_top:
stats->estimated_count = false;
stats->num_index_tuples = num_index_tuples;
stats->tuples_removed += tuples_removed;
+ pgstat_progress_update_param(PROGRESS_VACUUM_DEAD_TUPLES_VACUUMED, stats->tuples_removed);
+
/* hashvacuumcleanup will fill in num_pages */
return stats;
diff --git a/src/backend/access/heap/vacuumlazy.c b/src/backend/access/heap/vacuumlazy.c
index cd603e6aa4..f0cda3d162 100644
--- a/src/backend/access/heap/vacuumlazy.c
+++ b/src/backend/access/heap/vacuumlazy.c
@@ -167,6 +167,8 @@ typedef struct LVRelState
/* VACUUM operation's cutoff for freezing XIDs and MultiXactIds */
TransactionId FreezeLimit;
MultiXactId MultiXactCutoff;
+ /* VACCUM operation's longest index scan cycle */
+ int64 max_index_cycle_time;
/* Error reporting state */
char *relnamespace;
@@ -343,6 +345,7 @@ heap_vacuum_rel(Relation rel, VacuumParams *params,
pgstat_progress_start_command(PROGRESS_COMMAND_VACUUM,
RelationGetRelid(rel));
+ pgstat_progress_update_param(PROGRESS_VACUUM_LEADER_PID, MyProcPid);
vacuum_set_xid_limits(rel,
params->freeze_min_age,
@@ -2038,6 +2041,8 @@ static bool
lazy_vacuum_all_indexes(LVRelState *vacrel)
{
bool allindexes = true;
+ PGRUsage ru0;
+ int64 index_cycle_elapsed_ms = 0;
Assert(vacrel->nindexes > 0);
Assert(vacrel->do_index_vacuuming);
@@ -2056,6 +2061,10 @@ lazy_vacuum_all_indexes(LVRelState *vacrel)
pgstat_progress_update_param(PROGRESS_VACUUM_PHASE,
PROGRESS_VACUUM_PHASE_VACUUM_INDEX);
+ pgstat_progress_update_param(PROGRESS_VACUUM_TOTAL_INDEX_VACUUM, vacrel->nindexes);
+
+ pg_rusage_init(&ru0);
+
if (!ParallelVacuumIsActive(vacrel))
{
for (int idx = 0; idx < vacrel->nindexes; idx++)
@@ -2063,14 +2072,22 @@ lazy_vacuum_all_indexes(LVRelState *vacrel)
Relation indrel = vacrel->indrels[idx];
IndexBulkDeleteResult *istat = vacrel->indstats[idx];
+ /* Advertise the index that is being vacuumed and the vacuum order of the index */
+ pgstat_progress_update_param(PROGRESS_VACUUM_CURRENT_INDRELID, indrel->rd_id);
+ pgstat_progress_update_param(PROGRESS_VACUUM_INDEX_ORDINAL, idx + 1);
+
vacrel->indstats[idx] =
lazy_vacuum_one_index(indrel, istat, vacrel->old_live_tuples,
vacrel);
+ /* Advertise that we are done vacuuming the index */
+ pgstat_progress_update_param(PROGRESS_VACUUM_CURRENT_INDRELID, 0);
+
if (lazy_check_wraparound_failsafe(vacrel))
{
/* Wraparound emergency -- end current index scan */
allindexes = false;
+ pgstat_progress_update_param(PROGRESS_VACUUM_TOTAL_INDEX_VACUUM, 0);
break;
}
}
@@ -2085,8 +2102,10 @@ lazy_vacuum_all_indexes(LVRelState *vacrel)
* Do a postcheck to consider applying wraparound failsafe now. Note
* that parallel VACUUM only gets the precheck and this postcheck.
*/
- if (lazy_check_wraparound_failsafe(vacrel))
+ if (lazy_check_wraparound_failsafe(vacrel)) {
allindexes = false;
+ pgstat_progress_update_param(PROGRESS_VACUUM_TOTAL_INDEX_VACUUM, 0);
+ }
}
/*
@@ -2110,6 +2129,17 @@ lazy_vacuum_all_indexes(LVRelState *vacrel)
pgstat_progress_update_param(PROGRESS_VACUUM_NUM_INDEX_VACUUMS,
vacrel->num_index_scans);
+ /*
+ * Set the Maximum index vacuum cycle time.
+ *
+ * If this cycle took longer than any previous cycle, reset the max_index_cycle_time time.
+ */
+ index_cycle_elapsed_ms = pg_rusage_elapsed_ms(&ru0);
+ if (index_cycle_elapsed_ms > vacrel->max_index_cycle_time) {
+ vacrel->max_index_cycle_time = index_cycle_elapsed_ms;
+ pgstat_progress_update_param(PROGRESS_VACUUM_MAX_CYCLE_TIME, index_cycle_elapsed_ms);
+ }
+
return allindexes;
}
diff --git a/src/backend/access/nbtree/nbtree.c b/src/backend/access/nbtree/nbtree.c
index dfce06dc49..ed6e6ac42c 100644
--- a/src/backend/access/nbtree/nbtree.c
+++ b/src/backend/access/nbtree/nbtree.c
@@ -36,6 +36,8 @@
#include "utils/builtins.h"
#include "utils/index_selfuncs.h"
#include "utils/memutils.h"
+#include "commands/progress.h"
+#include "pgstat.h"
/*
@@ -1272,6 +1274,8 @@ backtrack:
nupdatable);
stats->tuples_removed += nhtidsdead;
+ pgstat_progress_update_param(PROGRESS_VACUUM_DEAD_TUPLES_VACUUMED, stats->tuples_removed);
+
/* must recompute maxoff */
maxoff = PageGetMaxOffsetNumber(page);
diff --git a/src/backend/access/spgist/spgvacuum.c b/src/backend/access/spgist/spgvacuum.c
index 76fb0374c4..6040761f33 100644
--- a/src/backend/access/spgist/spgvacuum.c
+++ b/src/backend/access/spgist/spgvacuum.c
@@ -27,6 +27,8 @@
#include "storage/indexfsm.h"
#include "storage/lmgr.h"
#include "utils/snapmgr.h"
+#include "commands/progress.h"
+#include "pgstat.h"
/* Entry in pending-list of TIDs we need to revisit */
@@ -160,6 +162,7 @@ vacuumLeafPage(spgBulkDeleteState *bds, Relation index, Buffer buffer,
bds->stats->tuples_removed += 1;
deletable[i] = true;
nDeletable++;
+ pgstat_progress_update_param(PROGRESS_VACUUM_DEAD_TUPLES_VACUUMED, bds->stats->tuples_removed);
}
else
{
@@ -430,6 +433,7 @@ vacuumLeafRoot(spgBulkDeleteState *bds, Relation index, Buffer buffer)
bds->stats->tuples_removed += 1;
toDelete[xlrec.nDelete] = i;
xlrec.nDelete++;
+ pgstat_progress_update_param(PROGRESS_VACUUM_DEAD_TUPLES_VACUUMED, bds->stats->tuples_removed);
}
else
{
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index 61b515cdb8..70f35092f6 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1124,9 +1124,20 @@ CREATE VIEW pg_stat_progress_vacuum AS
END AS phase,
S.param2 AS heap_blks_total, S.param3 AS heap_blks_scanned,
S.param4 AS heap_blks_vacuumed, S.param5 AS index_vacuum_count,
- S.param6 AS max_dead_tuples, S.param7 AS num_dead_tuples
+ S.param6 AS max_dead_tuples, S.param7 AS num_dead_tuples,
+ S.param11 AS total_index_vacuum, S.param12 AS max_index_vacuum_cycle_time
FROM pg_stat_get_progress_info('VACUUM') AS S
- LEFT JOIN pg_database D ON S.datid = D.oid;
+ LEFT JOIN pg_database D ON S.datid = D.oid
+ WHERE S.pid = S.param9; -- show vacuum progress for the leader PID only
+
+CREATE VIEW pg_stat_progress_vacuum_index AS
+ SELECT
+ S.pid AS pid, S.param9 AS leader_pid,
+ S.param8 AS indrelid, S.param10 vacuum_cycle_ordinal_position,
+ S.param13 index_rows_vacuumed
+ FROM pg_stat_get_progress_info('VACUUM') AS S
+ LEFT JOIN pg_database D ON S.datid = D.oid
+ WHERE S.param8 > 0; -- show vacuum progress for a PID that has advertised an index relid
CREATE VIEW pg_stat_progress_cluster AS
SELECT
diff --git a/src/backend/commands/vacuumparallel.c b/src/backend/commands/vacuumparallel.c
index 0d61c8ec74..983fc823bb 100644
--- a/src/backend/commands/vacuumparallel.c
+++ b/src/backend/commands/vacuumparallel.c
@@ -35,6 +35,7 @@
#include "storage/bufmgr.h"
#include "tcop/tcopprot.h"
#include "utils/lsyscache.h"
+#include "commands/progress.h"
/*
* DSM keys for parallel vacuum. Unlike other parallel execution code, since
@@ -206,7 +207,7 @@ static void parallel_vacuum_process_all_indexes(ParallelVacuumState *pvs, int nu
static void parallel_vacuum_process_safe_indexes(ParallelVacuumState *pvs);
static void parallel_vacuum_process_unsafe_indexes(ParallelVacuumState *pvs);
static void parallel_vacuum_process_one_index(ParallelVacuumState *pvs, Relation indrel,
- PVIndStats *indstats);
+ PVIndStats *indstats, int ordinal_position);
static bool parallel_vacuum_index_is_parallel_safe(Relation indrel, int num_index_scans,
bool vacuum);
static void parallel_vacuum_error_callback(void *arg);
@@ -754,7 +755,7 @@ parallel_vacuum_process_safe_indexes(ParallelVacuumState *pvs)
continue;
/* Do vacuum or cleanup of the index */
- parallel_vacuum_process_one_index(pvs, pvs->indrels[idx], indstats);
+ parallel_vacuum_process_one_index(pvs, pvs->indrels[idx], indstats, idx + 1);
}
/*
@@ -795,7 +796,7 @@ parallel_vacuum_process_unsafe_indexes(ParallelVacuumState *pvs)
continue;
/* Do vacuum or cleanup of the index */
- parallel_vacuum_process_one_index(pvs, pvs->indrels[i], indstats);
+ parallel_vacuum_process_one_index(pvs, pvs->indrels[i], indstats, i + 1);
}
/*
@@ -814,7 +815,7 @@ parallel_vacuum_process_unsafe_indexes(ParallelVacuumState *pvs)
*/
static void
parallel_vacuum_process_one_index(ParallelVacuumState *pvs, Relation indrel,
- PVIndStats *indstats)
+ PVIndStats *indstats, int ordinal_position)
{
IndexBulkDeleteResult *istat = NULL;
IndexBulkDeleteResult *istat_res;
@@ -842,7 +843,14 @@ parallel_vacuum_process_one_index(ParallelVacuumState *pvs, Relation indrel,
switch (indstats->status)
{
case PARALLEL_INDVAC_STATUS_NEED_BULKDELETE:
+ /* Advertise the index that is being vacuumed and the vacuum order of the index */
+ pgstat_progress_update_param(PROGRESS_VACUUM_CURRENT_INDRELID, indrel->rd_id);
+ pgstat_progress_update_param(PROGRESS_VACUUM_INDEX_ORDINAL, ordinal_position);
+
istat_res = vac_bulkdel_one_index(&ivinfo, istat, pvs->dead_items);
+
+ /* Advertise that we are done vacuuming the index */
+ pgstat_progress_update_param(PROGRESS_VACUUM_CURRENT_INDRELID, 0);
break;
case PARALLEL_INDVAC_STATUS_NEED_CLEANUP:
istat_res = vac_cleanup_one_index(&ivinfo, istat);
@@ -942,6 +950,7 @@ parallel_vacuum_main(dsm_segment *seg, shm_toc *toc)
int nindexes;
char *sharedquery;
ErrorContextCallback errcallback;
+ PGPROC *leader = MyProc->lockGroupLeader;
/*
* A parallel vacuum worker must have only PROC_IN_VACUUM flag since we
@@ -965,6 +974,10 @@ parallel_vacuum_main(dsm_segment *seg, shm_toc *toc)
*/
rel = table_open(shared->relid, ShareUpdateExclusiveLock);
+ pgstat_progress_start_command(PROGRESS_COMMAND_VACUUM,
+ RelationGetRelid(rel));
+ pgstat_progress_update_param(PROGRESS_VACUUM_LEADER_PID, leader->pid);
+
/*
* Open all indexes. indrels are sorted in order by OID, which should be
* matched to the leader's one.
@@ -1035,6 +1048,7 @@ parallel_vacuum_main(dsm_segment *seg, shm_toc *toc)
vac_close_indexes(nindexes, indrels, RowExclusiveLock);
table_close(rel, ShareUpdateExclusiveLock);
FreeAccessStrategy(pvs.bstrategy);
+ pgstat_progress_end_command();
}
/*
diff --git a/src/backend/utils/misc/pg_rusage.c b/src/backend/utils/misc/pg_rusage.c
index bb5d9e7c85..14362d7d65 100644
--- a/src/backend/utils/misc/pg_rusage.c
+++ b/src/backend/utils/misc/pg_rusage.c
@@ -19,6 +19,27 @@
#include "utils/pg_rusage.h"
+void rusage_adjust(const PGRUsage *ru0, PGRUsage *ru1);
+
+void
+rusage_adjust(const PGRUsage *ru0, PGRUsage *ru1)
+{
+ if (ru1->tv.tv_usec < ru0->tv.tv_usec)
+ {
+ ru1->tv.tv_sec--;
+ ru1->tv.tv_usec += 1000000;
+ }
+ if (ru1->ru.ru_stime.tv_usec < ru0->ru.ru_stime.tv_usec)
+ {
+ ru1->ru.ru_stime.tv_sec--;
+ ru1->ru.ru_stime.tv_usec += 1000000;
+ }
+ if (ru1->ru.ru_utime.tv_usec < ru0->ru.ru_utime.tv_usec)
+ {
+ ru1->ru.ru_utime.tv_sec--;
+ ru1->ru.ru_utime.tv_usec += 1000000;
+ }
+}
/*
* Initialize usage snapshot.
@@ -44,21 +65,7 @@ pg_rusage_show(const PGRUsage *ru0)
pg_rusage_init(&ru1);
- if (ru1.tv.tv_usec < ru0->tv.tv_usec)
- {
- ru1.tv.tv_sec--;
- ru1.tv.tv_usec += 1000000;
- }
- if (ru1.ru.ru_stime.tv_usec < ru0->ru.ru_stime.tv_usec)
- {
- ru1.ru.ru_stime.tv_sec--;
- ru1.ru.ru_stime.tv_usec += 1000000;
- }
- if (ru1.ru.ru_utime.tv_usec < ru0->ru.ru_utime.tv_usec)
- {
- ru1.ru.ru_utime.tv_sec--;
- ru1.ru.ru_utime.tv_usec += 1000000;
- }
+ rusage_adjust(ru0, &ru1);
snprintf(result, sizeof(result),
_("CPU: user: %d.%02d s, system: %d.%02d s, elapsed: %d.%02d s"),
@@ -71,3 +78,22 @@ pg_rusage_show(const PGRUsage *ru0)
return result;
}
+
+/*
+ * Compute elapsed time since ru0 usage snapshot, and return the
+ * value in Milliseconds.
+ */
+const int64
+pg_rusage_elapsed_ms(const PGRUsage *ru0)
+{
+ static int64 result;
+ PGRUsage ru1;
+
+ pg_rusage_init(&ru1);
+
+ rusage_adjust(ru0, &ru1);
+
+ result = (int64) (ru1.tv.tv_sec - ru0->tv.tv_sec) * 1000 + ((int64) (ru1.tv.tv_usec - ru0->tv.tv_usec) / 1000);
+
+ return result;
+}
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index d7bf16368b..1d25a6b345 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -18,13 +18,19 @@
#define PROGRESS_H
/* Progress parameters for (lazy) vacuum */
-#define PROGRESS_VACUUM_PHASE 0
-#define PROGRESS_VACUUM_TOTAL_HEAP_BLKS 1
-#define PROGRESS_VACUUM_HEAP_BLKS_SCANNED 2
-#define PROGRESS_VACUUM_HEAP_BLKS_VACUUMED 3
-#define PROGRESS_VACUUM_NUM_INDEX_VACUUMS 4
-#define PROGRESS_VACUUM_MAX_DEAD_TUPLES 5
-#define PROGRESS_VACUUM_NUM_DEAD_TUPLES 6
+#define PROGRESS_VACUUM_PHASE 0
+#define PROGRESS_VACUUM_TOTAL_HEAP_BLKS 1
+#define PROGRESS_VACUUM_HEAP_BLKS_SCANNED 2
+#define PROGRESS_VACUUM_HEAP_BLKS_VACUUMED 3
+#define PROGRESS_VACUUM_NUM_INDEX_VACUUMS 4
+#define PROGRESS_VACUUM_MAX_DEAD_TUPLES 5
+#define PROGRESS_VACUUM_NUM_DEAD_TUPLES 6
+#define PROGRESS_VACUUM_CURRENT_INDRELID 7
+#define PROGRESS_VACUUM_LEADER_PID 8
+#define PROGRESS_VACUUM_INDEX_ORDINAL 9
+#define PROGRESS_VACUUM_TOTAL_INDEX_VACUUM 10
+#define PROGRESS_VACUUM_MAX_CYCLE_TIME 11
+#define PROGRESS_VACUUM_DEAD_TUPLES_VACUUMED 12
/* Phases of vacuum (as advertised via PROGRESS_VACUUM_PHASE) */
#define PROGRESS_VACUUM_PHASE_SCAN_HEAP 1
diff --git a/src/include/utils/pg_rusage.h b/src/include/utils/pg_rusage.h
index c0def804ee..026f823155 100644
--- a/src/include/utils/pg_rusage.h
+++ b/src/include/utils/pg_rusage.h
@@ -33,5 +33,6 @@ typedef struct PGRUsage
extern void pg_rusage_init(PGRUsage *ru0);
extern const char *pg_rusage_show(const PGRUsage *ru0);
+extern const int64 pg_rusage_elapsed_ms(const PGRUsage *ru0);
#endif /* PG_RUSAGE_H */
--
2.32.0
^ permalink raw reply [nested|flat] 44+ messages in thread
* Re: Add index scan progress to pg_stat_progress_vacuum
@ 2021-12-29 17:51 Justin Pryzby <[email protected]>
parent: Imseih (AWS), Sami <[email protected]>
0 siblings, 1 reply; 44+ messages in thread
From: Justin Pryzby @ 2021-12-29 17:51 UTC (permalink / raw)
To: Imseih (AWS), Sami <[email protected]>; +Cc: Masahiko Sawada <[email protected]>; Peter Geoghegan <[email protected]>; Bossart, Nathan <[email protected]>; pgsql-hackers
http://cfbot.cputube.org/sami-imseih.html
You should run "make check" and update rules.out.
You should also use make check-world - usually something like:
make check-world -j4 >check-world.out 2>&1 ; echo ret $?
> indrelid: The relid of the index currently being vacuumed
I think it should be called indexrelid not indrelid, for consistency with
pg_index.
> S.param10 vacuum_cycle_ordinal_position,
> S.param13 index_rows_vacuumed
These should both say "AS" for consistency.
system_views.sql is using tabs, but should use spaces for consistency.
> #include "commands/progress.h"
The postgres convention is to alphabetize the includes.
> /* VACCUM operation's longest index scan cycle */
VACCUM => VACUUM
Ultimately you'll also need to update the docs.
^ permalink raw reply [nested|flat] 44+ messages in thread
* Re: Add index scan progress to pg_stat_progress_vacuum
@ 2021-12-31 05:59 Imseih (AWS), Sami <[email protected]>
parent: Justin Pryzby <[email protected]>
0 siblings, 0 replies; 44+ messages in thread
From: Imseih (AWS), Sami @ 2021-12-31 05:59 UTC (permalink / raw)
To: Justin Pryzby <[email protected]>; +Cc: Masahiko Sawada <[email protected]>; Peter Geoghegan <[email protected]>; Bossart, Nathan <[email protected]>; pgsql-hackers
Attaching the latest revision of the patch with the fixes suggested. Also ran make check and make check-world successfully.
On 12/29/21, 11:51 AM, "Justin Pryzby" <[email protected]> wrote:
CAUTION: This email originated from outside of the organization. Do not click links or open attachments unless you can confirm the sender and know the content is safe.
http://cfbot.cputube.org/sami-imseih.html
You should run "make check" and update rules.out.
You should also use make check-world - usually something like:
make check-world -j4 >check-world.out 2>&1 ; echo ret $?
> indrelid: The relid of the index currently being vacuumed
I think it should be called indexrelid not indrelid, for consistency with
pg_index.
> S.param10 vacuum_cycle_ordinal_position,
> S.param13 index_rows_vacuumed
These should both say "AS" for consistency.
system_views.sql is using tabs, but should use spaces for consistency.
> #include "commands/progress.h"
The postgres convention is to alphabetize the includes.
> /* VACCUM operation's longest index scan cycle */
VACCUM => VACUUM
Ultimately you'll also need to update the docs.
Attachments:
[application/octet-stream] 0001-Add-index-scan-progress-to-pg_stat_progress_vacuum.patch (18.1K, ../../[email protected]/2-0001-Add-index-scan-progress-to-pg_stat_progress_vacuum.patch)
download | inline diff:
From 62af0b730ff30f9d4ff5c5b9663bb062440a9fb7 Mon Sep 17 00:00:00 2001
From: "Imseih (AWS), Sami" <[email protected]>
Date: Fri, 31 Dec 2021 04:43:31 +0000
Subject: [PATCH 1/1] Add index scan progress to pg_stat_progress_vacuum.
Expose progress for the "vacuuming indexes" phase
of a VACUUM operation.
Author: Sami Imseih, based on suggestions by Nathan Bossart, Peter Geoghegan and Masahiko Sawada
Reviewed by: Nathan Bossart, Justin Pryzby
Discussion: https://www.postgresql.org/message-id/5478DFCD-2333-401A-B2F0-0D186AB09228%40amazon.com
---
src/backend/access/gin/ginvacuum.c | 4 ++
src/backend/access/gist/gistvacuum.c | 4 ++
src/backend/access/hash/hash.c | 2 +
src/backend/access/heap/vacuumlazy.c | 32 ++++++++++++++-
src/backend/access/nbtree/nbtree.c | 2 +
src/backend/access/spgist/spgvacuum.c | 4 ++
src/backend/catalog/system_views.sql | 15 ++++++-
src/backend/commands/vacuumparallel.c | 22 +++++++++--
src/backend/utils/misc/pg_rusage.c | 56 ++++++++++++++++++++-------
src/include/commands/progress.h | 20 ++++++----
src/include/utils/pg_rusage.h | 1 +
src/test/regress/expected/rules.out | 15 ++++++-
12 files changed, 146 insertions(+), 31 deletions(-)
diff --git a/src/backend/access/gin/ginvacuum.c b/src/backend/access/gin/ginvacuum.c
index a276eb020b..90706be135 100644
--- a/src/backend/access/gin/ginvacuum.c
+++ b/src/backend/access/gin/ginvacuum.c
@@ -17,8 +17,10 @@
#include "access/gin_private.h"
#include "access/ginxlog.h"
#include "access/xloginsert.h"
+#include "commands/progress.h"
#include "commands/vacuum.h"
#include "miscadmin.h"
+#include "pgstat.h"
#include "postmaster/autovacuum.h"
#include "storage/indexfsm.h"
#include "storage/lmgr.h"
@@ -60,6 +62,8 @@ ginVacuumItemPointers(GinVacuumState *gvs, ItemPointerData *items,
if (gvs->callback(items + i, gvs->callback_state))
{
gvs->result->tuples_removed += 1;
+ pgstat_progress_update_param(PROGRESS_VACUUM_DEAD_TUPLES_VACUUMED, gvs->result->tuples_removed);
+
if (!tmpitems)
{
/*
diff --git a/src/backend/access/gist/gistvacuum.c b/src/backend/access/gist/gistvacuum.c
index 0663193531..9241175329 100644
--- a/src/backend/access/gist/gistvacuum.c
+++ b/src/backend/access/gist/gistvacuum.c
@@ -17,9 +17,11 @@
#include "access/genam.h"
#include "access/gist_private.h"
#include "access/transam.h"
+#include "commands/progress.h"
#include "commands/vacuum.h"
#include "lib/integerset.h"
#include "miscadmin.h"
+#include "pgstat.h"
#include "storage/indexfsm.h"
#include "storage/lmgr.h"
#include "utils/memutils.h"
@@ -375,6 +377,8 @@ restart:
END_CRIT_SECTION();
vstate->stats->tuples_removed += ntodelete;
+ pgstat_progress_update_param(PROGRESS_VACUUM_DEAD_TUPLES_VACUUMED, vstate->stats->tuples_removed);
+
/* must recompute maxoff */
maxoff = PageGetMaxOffsetNumber(page);
}
diff --git a/src/backend/access/hash/hash.c b/src/backend/access/hash/hash.c
index 81c7da7ec6..45d2942c9e 100644
--- a/src/backend/access/hash/hash.c
+++ b/src/backend/access/hash/hash.c
@@ -631,6 +631,8 @@ loop_top:
stats->estimated_count = false;
stats->num_index_tuples = num_index_tuples;
stats->tuples_removed += tuples_removed;
+ pgstat_progress_update_param(PROGRESS_VACUUM_DEAD_TUPLES_VACUUMED, stats->tuples_removed);
+
/* hashvacuumcleanup will fill in num_pages */
return stats;
diff --git a/src/backend/access/heap/vacuumlazy.c b/src/backend/access/heap/vacuumlazy.c
index cd603e6aa4..5b45ee3e2e 100644
--- a/src/backend/access/heap/vacuumlazy.c
+++ b/src/backend/access/heap/vacuumlazy.c
@@ -167,6 +167,8 @@ typedef struct LVRelState
/* VACUUM operation's cutoff for freezing XIDs and MultiXactIds */
TransactionId FreezeLimit;
MultiXactId MultiXactCutoff;
+ /* VACUUM operation's longest index scan cycle */
+ int64 max_index_cycle_time;
/* Error reporting state */
char *relnamespace;
@@ -343,6 +345,7 @@ heap_vacuum_rel(Relation rel, VacuumParams *params,
pgstat_progress_start_command(PROGRESS_COMMAND_VACUUM,
RelationGetRelid(rel));
+ pgstat_progress_update_param(PROGRESS_VACUUM_LEADER_PID, MyProcPid);
vacuum_set_xid_limits(rel,
params->freeze_min_age,
@@ -2038,6 +2041,8 @@ static bool
lazy_vacuum_all_indexes(LVRelState *vacrel)
{
bool allindexes = true;
+ PGRUsage ru0;
+ int64 index_cycle_elapsed_ms = 0;
Assert(vacrel->nindexes > 0);
Assert(vacrel->do_index_vacuuming);
@@ -2056,6 +2061,10 @@ lazy_vacuum_all_indexes(LVRelState *vacrel)
pgstat_progress_update_param(PROGRESS_VACUUM_PHASE,
PROGRESS_VACUUM_PHASE_VACUUM_INDEX);
+ pgstat_progress_update_param(PROGRESS_VACUUM_TOTAL_INDEX_VACUUM, vacrel->nindexes);
+
+ pg_rusage_init(&ru0);
+
if (!ParallelVacuumIsActive(vacrel))
{
for (int idx = 0; idx < vacrel->nindexes; idx++)
@@ -2063,14 +2072,22 @@ lazy_vacuum_all_indexes(LVRelState *vacrel)
Relation indrel = vacrel->indrels[idx];
IndexBulkDeleteResult *istat = vacrel->indstats[idx];
+ /* Advertise the index that is being vacuumed and the vacuum order of the index */
+ pgstat_progress_update_param(PROGRESS_VACUUM_CURRENT_INDRELID, indrel->rd_id);
+ pgstat_progress_update_param(PROGRESS_VACUUM_INDEX_ORDINAL, idx + 1);
+
vacrel->indstats[idx] =
lazy_vacuum_one_index(indrel, istat, vacrel->old_live_tuples,
vacrel);
+ /* Advertise that we are done vacuuming the index */
+ pgstat_progress_update_param(PROGRESS_VACUUM_CURRENT_INDRELID, 0);
+
if (lazy_check_wraparound_failsafe(vacrel))
{
/* Wraparound emergency -- end current index scan */
allindexes = false;
+ pgstat_progress_update_param(PROGRESS_VACUUM_TOTAL_INDEX_VACUUM, 0);
break;
}
}
@@ -2085,8 +2102,10 @@ lazy_vacuum_all_indexes(LVRelState *vacrel)
* Do a postcheck to consider applying wraparound failsafe now. Note
* that parallel VACUUM only gets the precheck and this postcheck.
*/
- if (lazy_check_wraparound_failsafe(vacrel))
+ if (lazy_check_wraparound_failsafe(vacrel)) {
allindexes = false;
+ pgstat_progress_update_param(PROGRESS_VACUUM_TOTAL_INDEX_VACUUM, 0);
+ }
}
/*
@@ -2110,6 +2129,17 @@ lazy_vacuum_all_indexes(LVRelState *vacrel)
pgstat_progress_update_param(PROGRESS_VACUUM_NUM_INDEX_VACUUMS,
vacrel->num_index_scans);
+ /*
+ * Set the Maximum index vacuum cycle time.
+ *
+ * If this cycle took longer than any previous cycle, reset the max_index_cycle_time time.
+ */
+ index_cycle_elapsed_ms = pg_rusage_elapsed_ms(&ru0);
+ if (index_cycle_elapsed_ms > vacrel->max_index_cycle_time) {
+ vacrel->max_index_cycle_time = index_cycle_elapsed_ms;
+ pgstat_progress_update_param(PROGRESS_VACUUM_MAX_CYCLE_TIME, index_cycle_elapsed_ms);
+ }
+
return allindexes;
}
diff --git a/src/backend/access/nbtree/nbtree.c b/src/backend/access/nbtree/nbtree.c
index dfce06dc49..1481fb41ea 100644
--- a/src/backend/access/nbtree/nbtree.c
+++ b/src/backend/access/nbtree/nbtree.c
@@ -1272,6 +1272,8 @@ backtrack:
nupdatable);
stats->tuples_removed += nhtidsdead;
+ pgstat_progress_update_param(PROGRESS_VACUUM_DEAD_TUPLES_VACUUMED, stats->tuples_removed);
+
/* must recompute maxoff */
maxoff = PageGetMaxOffsetNumber(page);
diff --git a/src/backend/access/spgist/spgvacuum.c b/src/backend/access/spgist/spgvacuum.c
index 76fb0374c4..989a50ef86 100644
--- a/src/backend/access/spgist/spgvacuum.c
+++ b/src/backend/access/spgist/spgvacuum.c
@@ -21,8 +21,10 @@
#include "access/transam.h"
#include "access/xloginsert.h"
#include "catalog/storage_xlog.h"
+#include "commands/progress.h"
#include "commands/vacuum.h"
#include "miscadmin.h"
+#include "pgstat.h"
#include "storage/bufmgr.h"
#include "storage/indexfsm.h"
#include "storage/lmgr.h"
@@ -160,6 +162,7 @@ vacuumLeafPage(spgBulkDeleteState *bds, Relation index, Buffer buffer,
bds->stats->tuples_removed += 1;
deletable[i] = true;
nDeletable++;
+ pgstat_progress_update_param(PROGRESS_VACUUM_DEAD_TUPLES_VACUUMED, bds->stats->tuples_removed);
}
else
{
@@ -430,6 +433,7 @@ vacuumLeafRoot(spgBulkDeleteState *bds, Relation index, Buffer buffer)
bds->stats->tuples_removed += 1;
toDelete[xlrec.nDelete] = i;
xlrec.nDelete++;
+ pgstat_progress_update_param(PROGRESS_VACUUM_DEAD_TUPLES_VACUUMED, bds->stats->tuples_removed);
}
else
{
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index 61b515cdb8..788a1824cc 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1124,9 +1124,20 @@ CREATE VIEW pg_stat_progress_vacuum AS
END AS phase,
S.param2 AS heap_blks_total, S.param3 AS heap_blks_scanned,
S.param4 AS heap_blks_vacuumed, S.param5 AS index_vacuum_count,
- S.param6 AS max_dead_tuples, S.param7 AS num_dead_tuples
+ S.param6 AS max_dead_tuples, S.param7 AS num_dead_tuples,
+ S.param11 AS total_index_vacuum, S.param12 AS max_index_vacuum_cycle_time
FROM pg_stat_get_progress_info('VACUUM') AS S
- LEFT JOIN pg_database D ON S.datid = D.oid;
+ LEFT JOIN pg_database D ON S.datid = D.oid
+ WHERE S.pid = S.param9; -- show vacuum progress for the leader PID only
+
+CREATE VIEW pg_stat_progress_vacuum_index AS
+ SELECT
+ S.pid AS pid, S.param9 AS leader_pid,
+ S.param8 AS indexrelid, S.param10 AS vacuum_cycle_ordinal_position,
+ S.param13 AS index_rows_vacuumed
+ FROM pg_stat_get_progress_info('VACUUM') AS S
+ LEFT JOIN pg_database D ON S.datid = D.oid
+ WHERE S.param8 > 0; -- show vacuum progress for a PID that has advertised an index relid
CREATE VIEW pg_stat_progress_cluster AS
SELECT
diff --git a/src/backend/commands/vacuumparallel.c b/src/backend/commands/vacuumparallel.c
index 0d61c8ec74..1eda5785e5 100644
--- a/src/backend/commands/vacuumparallel.c
+++ b/src/backend/commands/vacuumparallel.c
@@ -29,6 +29,7 @@
#include "access/amapi.h"
#include "access/table.h"
#include "catalog/index.h"
+#include "commands/progress.h"
#include "commands/vacuum.h"
#include "optimizer/paths.h"
#include "pgstat.h"
@@ -206,7 +207,7 @@ static void parallel_vacuum_process_all_indexes(ParallelVacuumState *pvs, int nu
static void parallel_vacuum_process_safe_indexes(ParallelVacuumState *pvs);
static void parallel_vacuum_process_unsafe_indexes(ParallelVacuumState *pvs);
static void parallel_vacuum_process_one_index(ParallelVacuumState *pvs, Relation indrel,
- PVIndStats *indstats);
+ PVIndStats *indstats, int ordinal_position);
static bool parallel_vacuum_index_is_parallel_safe(Relation indrel, int num_index_scans,
bool vacuum);
static void parallel_vacuum_error_callback(void *arg);
@@ -754,7 +755,7 @@ parallel_vacuum_process_safe_indexes(ParallelVacuumState *pvs)
continue;
/* Do vacuum or cleanup of the index */
- parallel_vacuum_process_one_index(pvs, pvs->indrels[idx], indstats);
+ parallel_vacuum_process_one_index(pvs, pvs->indrels[idx], indstats, idx + 1);
}
/*
@@ -795,7 +796,7 @@ parallel_vacuum_process_unsafe_indexes(ParallelVacuumState *pvs)
continue;
/* Do vacuum or cleanup of the index */
- parallel_vacuum_process_one_index(pvs, pvs->indrels[i], indstats);
+ parallel_vacuum_process_one_index(pvs, pvs->indrels[i], indstats, i + 1);
}
/*
@@ -814,7 +815,7 @@ parallel_vacuum_process_unsafe_indexes(ParallelVacuumState *pvs)
*/
static void
parallel_vacuum_process_one_index(ParallelVacuumState *pvs, Relation indrel,
- PVIndStats *indstats)
+ PVIndStats *indstats, int ordinal_position)
{
IndexBulkDeleteResult *istat = NULL;
IndexBulkDeleteResult *istat_res;
@@ -842,7 +843,14 @@ parallel_vacuum_process_one_index(ParallelVacuumState *pvs, Relation indrel,
switch (indstats->status)
{
case PARALLEL_INDVAC_STATUS_NEED_BULKDELETE:
+ /* Advertise the index that is being vacuumed and the vacuum order of the index */
+ pgstat_progress_update_param(PROGRESS_VACUUM_CURRENT_INDRELID, indrel->rd_id);
+ pgstat_progress_update_param(PROGRESS_VACUUM_INDEX_ORDINAL, ordinal_position);
+
istat_res = vac_bulkdel_one_index(&ivinfo, istat, pvs->dead_items);
+
+ /* Advertise that we are done vacuuming the index */
+ pgstat_progress_update_param(PROGRESS_VACUUM_CURRENT_INDRELID, 0);
break;
case PARALLEL_INDVAC_STATUS_NEED_CLEANUP:
istat_res = vac_cleanup_one_index(&ivinfo, istat);
@@ -942,6 +950,7 @@ parallel_vacuum_main(dsm_segment *seg, shm_toc *toc)
int nindexes;
char *sharedquery;
ErrorContextCallback errcallback;
+ PGPROC *leader = MyProc->lockGroupLeader;
/*
* A parallel vacuum worker must have only PROC_IN_VACUUM flag since we
@@ -965,6 +974,10 @@ parallel_vacuum_main(dsm_segment *seg, shm_toc *toc)
*/
rel = table_open(shared->relid, ShareUpdateExclusiveLock);
+ pgstat_progress_start_command(PROGRESS_COMMAND_VACUUM,
+ RelationGetRelid(rel));
+ pgstat_progress_update_param(PROGRESS_VACUUM_LEADER_PID, leader->pid);
+
/*
* Open all indexes. indrels are sorted in order by OID, which should be
* matched to the leader's one.
@@ -1035,6 +1048,7 @@ parallel_vacuum_main(dsm_segment *seg, shm_toc *toc)
vac_close_indexes(nindexes, indrels, RowExclusiveLock);
table_close(rel, ShareUpdateExclusiveLock);
FreeAccessStrategy(pvs.bstrategy);
+ pgstat_progress_end_command();
}
/*
diff --git a/src/backend/utils/misc/pg_rusage.c b/src/backend/utils/misc/pg_rusage.c
index bb5d9e7c85..14362d7d65 100644
--- a/src/backend/utils/misc/pg_rusage.c
+++ b/src/backend/utils/misc/pg_rusage.c
@@ -19,6 +19,27 @@
#include "utils/pg_rusage.h"
+void rusage_adjust(const PGRUsage *ru0, PGRUsage *ru1);
+
+void
+rusage_adjust(const PGRUsage *ru0, PGRUsage *ru1)
+{
+ if (ru1->tv.tv_usec < ru0->tv.tv_usec)
+ {
+ ru1->tv.tv_sec--;
+ ru1->tv.tv_usec += 1000000;
+ }
+ if (ru1->ru.ru_stime.tv_usec < ru0->ru.ru_stime.tv_usec)
+ {
+ ru1->ru.ru_stime.tv_sec--;
+ ru1->ru.ru_stime.tv_usec += 1000000;
+ }
+ if (ru1->ru.ru_utime.tv_usec < ru0->ru.ru_utime.tv_usec)
+ {
+ ru1->ru.ru_utime.tv_sec--;
+ ru1->ru.ru_utime.tv_usec += 1000000;
+ }
+}
/*
* Initialize usage snapshot.
@@ -44,21 +65,7 @@ pg_rusage_show(const PGRUsage *ru0)
pg_rusage_init(&ru1);
- if (ru1.tv.tv_usec < ru0->tv.tv_usec)
- {
- ru1.tv.tv_sec--;
- ru1.tv.tv_usec += 1000000;
- }
- if (ru1.ru.ru_stime.tv_usec < ru0->ru.ru_stime.tv_usec)
- {
- ru1.ru.ru_stime.tv_sec--;
- ru1.ru.ru_stime.tv_usec += 1000000;
- }
- if (ru1.ru.ru_utime.tv_usec < ru0->ru.ru_utime.tv_usec)
- {
- ru1.ru.ru_utime.tv_sec--;
- ru1.ru.ru_utime.tv_usec += 1000000;
- }
+ rusage_adjust(ru0, &ru1);
snprintf(result, sizeof(result),
_("CPU: user: %d.%02d s, system: %d.%02d s, elapsed: %d.%02d s"),
@@ -71,3 +78,22 @@ pg_rusage_show(const PGRUsage *ru0)
return result;
}
+
+/*
+ * Compute elapsed time since ru0 usage snapshot, and return the
+ * value in Milliseconds.
+ */
+const int64
+pg_rusage_elapsed_ms(const PGRUsage *ru0)
+{
+ static int64 result;
+ PGRUsage ru1;
+
+ pg_rusage_init(&ru1);
+
+ rusage_adjust(ru0, &ru1);
+
+ result = (int64) (ru1.tv.tv_sec - ru0->tv.tv_sec) * 1000 + ((int64) (ru1.tv.tv_usec - ru0->tv.tv_usec) / 1000);
+
+ return result;
+}
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index d7bf16368b..1d25a6b345 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -18,13 +18,19 @@
#define PROGRESS_H
/* Progress parameters for (lazy) vacuum */
-#define PROGRESS_VACUUM_PHASE 0
-#define PROGRESS_VACUUM_TOTAL_HEAP_BLKS 1
-#define PROGRESS_VACUUM_HEAP_BLKS_SCANNED 2
-#define PROGRESS_VACUUM_HEAP_BLKS_VACUUMED 3
-#define PROGRESS_VACUUM_NUM_INDEX_VACUUMS 4
-#define PROGRESS_VACUUM_MAX_DEAD_TUPLES 5
-#define PROGRESS_VACUUM_NUM_DEAD_TUPLES 6
+#define PROGRESS_VACUUM_PHASE 0
+#define PROGRESS_VACUUM_TOTAL_HEAP_BLKS 1
+#define PROGRESS_VACUUM_HEAP_BLKS_SCANNED 2
+#define PROGRESS_VACUUM_HEAP_BLKS_VACUUMED 3
+#define PROGRESS_VACUUM_NUM_INDEX_VACUUMS 4
+#define PROGRESS_VACUUM_MAX_DEAD_TUPLES 5
+#define PROGRESS_VACUUM_NUM_DEAD_TUPLES 6
+#define PROGRESS_VACUUM_CURRENT_INDRELID 7
+#define PROGRESS_VACUUM_LEADER_PID 8
+#define PROGRESS_VACUUM_INDEX_ORDINAL 9
+#define PROGRESS_VACUUM_TOTAL_INDEX_VACUUM 10
+#define PROGRESS_VACUUM_MAX_CYCLE_TIME 11
+#define PROGRESS_VACUUM_DEAD_TUPLES_VACUUMED 12
/* Phases of vacuum (as advertised via PROGRESS_VACUUM_PHASE) */
#define PROGRESS_VACUUM_PHASE_SCAN_HEAP 1
diff --git a/src/include/utils/pg_rusage.h b/src/include/utils/pg_rusage.h
index c0def804ee..026f823155 100644
--- a/src/include/utils/pg_rusage.h
+++ b/src/include/utils/pg_rusage.h
@@ -33,5 +33,6 @@ typedef struct PGRUsage
extern void pg_rusage_init(PGRUsage *ru0);
extern const char *pg_rusage_show(const PGRUsage *ru0);
+extern const int64 pg_rusage_elapsed_ms(const PGRUsage *ru0);
#endif /* PG_RUSAGE_H */
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b58b062b10..22c54b36f1 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -2024,9 +2024,20 @@ pg_stat_progress_vacuum| SELECT s.pid,
s.param4 AS heap_blks_vacuumed,
s.param5 AS index_vacuum_count,
s.param6 AS max_dead_tuples,
- s.param7 AS num_dead_tuples
+ s.param7 AS num_dead_tuples,
+ s.param11 AS total_index_vacuum,
+ s.param12 AS max_index_vacuum_cycle_time
FROM (pg_stat_get_progress_info('VACUUM'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
- LEFT JOIN pg_database d ON ((s.datid = d.oid)));
+ LEFT JOIN pg_database d ON ((s.datid = d.oid)))
+ WHERE (s.pid = s.param9);
+pg_stat_progress_vacuum_index| SELECT s.pid,
+ s.param9 AS leader_pid,
+ s.param8 AS indexrelid,
+ s.param10 AS vacuum_cycle_ordinal_position,
+ s.param13 AS index_rows_vacuumed
+ FROM (pg_stat_get_progress_info('VACUUM'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
+ LEFT JOIN pg_database d ON ((s.datid = d.oid)))
+ WHERE (s.param8 > 0);
pg_stat_replication| SELECT s.pid,
s.usesysid,
u.rolname AS usename,
--
2.32.0
^ permalink raw reply [nested|flat] 44+ messages in thread
* Re: Add index scan progress to pg_stat_progress_vacuum
@ 2022-03-25 14:54 Masahiko Sawada <[email protected]>
0 siblings, 1 reply; 44+ messages in thread
From: Masahiko Sawada @ 2022-03-25 14:54 UTC (permalink / raw)
To: Imseih (AWS), Sami <[email protected]>; +Cc: Nathan Bossart <[email protected]>; Peter Geoghegan <[email protected]>; Bossart, Nathan <[email protected]>; Justin Pryzby <[email protected]>; pgsql-hackers
On Wed, Mar 23, 2022 at 6:57 AM Imseih (AWS), Sami <[email protected]> wrote:
>
> > Can the leader pass a callback that checks PVIndStats to ambulkdelete
> > an amvacuumcleanup callbacks? I think that in the passed callback, the
> > leader checks if the number of processed indexes and updates its
> > progress information if the current progress needs to be updated.
>
> Thanks for the suggestion.
>
> I looked at this option a but today and found that passing the callback
> will also require signature changes to the ambulkdelete and
> amvacuumcleanup routines.
I think it would not be a critical problem since it's a new feature.
>
> This will also require us to check after x pages have been
> scanned inside vacuumscan and vacuumcleanup. After x pages
> the callback can then update the leaders progress.
> I am not sure if adding additional complexity to the scan/cleanup path
> is justified for what this patch is attempting to do.
>
> There will also be a lag of the leader updating the progress as it
> must scan x amount of pages before updating. Obviously, the more
> Pages to the scan, the longer the lag will be.
Fair points.
On the other hand, the approach of the current patch requires more
memory for progress tracking, which could fail, e.g., due to running
out of hashtable entries. I think that it would be worse that the
parallel operation failed to start due to not being able to track the
progress than the above concerns you mentioned such as introducing
additional complexity and a possible lag of progress updates. So if we
go with the current approach, I think we need to make sure enough (and
not too many) hash table entries.
Regards,
--
Masahiko Sawada
EDB: https://www.enterprisedb.com/
^ permalink raw reply [nested|flat] 44+ messages in thread
* Re: Add index scan progress to pg_stat_progress_vacuum
@ 2022-03-29 12:08 Imseih (AWS), Sami <[email protected]>
parent: Masahiko Sawada <[email protected]>
0 siblings, 1 reply; 44+ messages in thread
From: Imseih (AWS), Sami @ 2022-03-29 12:08 UTC (permalink / raw)
To: Masahiko Sawada <[email protected]>; +Cc: Nathan Bossart <[email protected]>; Peter Geoghegan <[email protected]>; Bossart, Nathan <[email protected]>; Justin Pryzby <[email protected]>; pgsql-hackers
Sorry for the late reply.
> additional complexity and a possible lag of progress updates. So if we
> go with the current approach, I think we need to make sure enough (and
> not too many) hash table entries.
The hash table can be set 4 times the size of
max_worker_processes which should give more than
enough padding.
Note that max_parallel_maintenance_workers
is what should be used, but since it's dynamic, it cannot
be used to determine the size of shared memory.
Regards,
---
Sami Imseih
Amazon Web Services
^ permalink raw reply [nested|flat] 44+ messages in thread
* Re: Add index scan progress to pg_stat_progress_vacuum
@ 2022-03-29 12:25 Imseih (AWS), Sami <[email protected]>
parent: Imseih (AWS), Sami <[email protected]>
0 siblings, 1 reply; 44+ messages in thread
From: Imseih (AWS), Sami @ 2022-03-29 12:25 UTC (permalink / raw)
To: Andres Freund <[email protected]>; +Cc: Nathan Bossart <[email protected]>; Peter Geoghegan <[email protected]>; Bossart, Nathan <[email protected]>; Justin Pryzby <[email protected]>; pgsql-hackers; Masahiko Sawada <[email protected]>
> I think that's an absolute no-go. Adding locking to progress reporting,
> particularly a single central lwlock, is going to *vastly* increase the
> overhead incurred by progress reporting.
Sorry for the late reply.
The usage of the shared memory will be limited
to PARALLEL maintenance operations. For now,
it will only be populated for parallel vacuums.
Autovacuum for example will not be required to
populate this shared memory.
Regards,
---
Sami Imseih
Amazon Web Services
^ permalink raw reply [nested|flat] 44+ messages in thread
* Re: Add index scan progress to pg_stat_progress_vacuum
@ 2022-05-05 19:26 Imseih (AWS), Sami <[email protected]>
parent: Imseih (AWS), Sami <[email protected]>
0 siblings, 2 replies; 44+ messages in thread
From: Imseih (AWS), Sami @ 2022-05-05 19:26 UTC (permalink / raw)
To: Masahiko Sawada <[email protected]>; +Cc: Robert Haas <[email protected]>; Bossart, Nathan <[email protected]>; Andres Freund <[email protected]>; Peter Geoghegan <[email protected]>; Justin Pryzby <[email protected]>; pgsql-hackers
Thank you for the feedback!
> I think we can pass the progress update function to
> WaitForParallelWorkersToFinish(), which seems simpler. And we can call
Directly passing the callback to WaitForParallelWorkersToFinish
will require us to modify the function signature.
To me, it seemed simpler and touches less code to have
the caller set the callback in the ParallelContext.
> the function after updating the index status to
> PARALLEL_INDVAC_STATUS_COMPLETED.
I also like this better. Will make the change.
> BTW, currently we don't need a lock for touching index status since
> each worker touches different indexes. But after this patch, the
> leader will touch all index status, do we need a lock for that?
I do not think locking is needed here. The leader and workers
will continue to touch different indexes to update the status.
However, if the process is a leader, it will call the function
which will go through indstats and count how many
Indexes have a status of PARALLEL_INDVAC_STATUS_COMPLETED.
This value is then reported to the leaders backend only.
Regards,
Sami Imseih
Amazon Web Services
^ permalink raw reply [nested|flat] 44+ messages in thread
* Re: Add index scan progress to pg_stat_progress_vacuum
@ 2022-05-26 13:41 Imseih (AWS), Sami <[email protected]>
parent: Imseih (AWS), Sami <[email protected]>
1 sibling, 0 replies; 44+ messages in thread
From: Imseih (AWS), Sami @ 2022-05-26 13:41 UTC (permalink / raw)
To: Masahiko Sawada <[email protected]>; +Cc: Robert Haas <[email protected]>; Bossart, Nathan <[email protected]>; Andres Freund <[email protected]>; Peter Geoghegan <[email protected]>; Justin Pryzby <[email protected]>; pgsql-hackers
> the function after updating the index status to
> PARALLEL_INDVAC_STATUS_COMPLETED.
> I also like this better. Will make the change.
I updated the patch. The progress function is called after
updating index status to PARALLEL_INDVAC_STATUS_COMPLETED.
I believe all comments have been addressed at this point.
Regards,
Sami Imseih
Amazon Web Services
Attachments:
[application/octet-stream] v11-0001-Add-progress-reporting-callback-to-ParallelConte.patch (2.6K, ../../[email protected]/2-v11-0001-Add-progress-reporting-callback-to-ParallelConte.patch)
download | inline diff:
From 19b1a9b4814fdf411a34a0ca86ce224079c857cc Mon Sep 17 00:00:00 2001
From: "Imseih (AWS)" <[email protected]>
Date: Thu, 26 May 2022 07:45:26 -0500
Subject: [PATCH v11 1/2] Add progress reporting callback to ParallelContext
The purpose of supporting a progress reporting
callback in ParallelContext is to allow for the
leader process to report progress while waiting
for workers to complete.
The first use-case for this is to report index
progress in pg_stat_progress_vacuum.
Author: Sami Imseih, based on suggestions by Nathan Bossart, Peter Geoghegan and Masahiko Sawada
Reviewed by: Nathan Bossart, Masahiko Sawada
Discussion: https://www.postgresql.org/message-id/flat/[email protected]
---
src/backend/access/transam/parallel.c | 16 ++++++++++++++++
src/include/access/parallel.h | 5 +++++
2 files changed, 21 insertions(+)
diff --git a/src/backend/access/transam/parallel.c b/src/backend/access/transam/parallel.c
index df0cd77..bfe3275 100644
--- a/src/backend/access/transam/parallel.c
+++ b/src/backend/access/transam/parallel.c
@@ -774,6 +774,22 @@ WaitForParallelWorkersToFinish(ParallelContext *pcxt)
*/
CHECK_FOR_INTERRUPTS();
+ /*
+ * We call the parallel progress callback while
+ * waiting for the parallel workers to finish.
+ * This is to ensure that the leader keeps
+ * updating progress when waiting for
+ * parallel workers to finish.
+ *
+ * We must ensure that pcxt->parallel_progress_callback
+ * is set before calling as not all parallel
+ * operations will set a callback.
+ */
+ if (pcxt->parallel_progress_callback)
+ {
+ pcxt->parallel_progress_callback(pcxt->parallel_progress_callback_arg);
+ }
+
for (i = 0; i < pcxt->nworkers_launched; ++i)
{
/*
diff --git a/src/include/access/parallel.h b/src/include/access/parallel.h
index 983841d..53a3d13 100644
--- a/src/include/access/parallel.h
+++ b/src/include/access/parallel.h
@@ -20,6 +20,9 @@
#include "storage/shm_mq.h"
#include "storage/shm_toc.h"
+/* progress callback definition */
+typedef void (*ParallelProgressCallback) (void *parallel_progress_callback_state);
+
typedef void (*parallel_worker_main_type) (dsm_segment *seg, shm_toc *toc);
typedef struct ParallelWorkerInfo
@@ -46,6 +49,8 @@ typedef struct ParallelContext
ParallelWorkerInfo *worker;
int nknown_attached_workers;
bool *known_attached_workers;
+ ParallelProgressCallback parallel_progress_callback;
+ void *parallel_progress_callback_arg;
} ParallelContext;
typedef struct ParallelWorkerContext
--
2.32.1 (Apple Git-133)
[application/octet-stream] v11-0002-Show-progress-for-index-vacuums.patch (9.5K, ../../[email protected]/3-v11-0002-Show-progress-for-index-vacuums.patch)
download | inline diff:
From 5d0c7f7f761316d1089b010bc750cf36803b53e3 Mon Sep 17 00:00:00 2001
From: "Imseih (AWS)" <[email protected]>
Date: Thu, 26 May 2022 08:02:11 -0500
Subject: [PATCH v11 2/2] Show progress for index vacuums
Add 2 new columns to pg_stat_progress_vacuum. The columns are
indexes_total as the total indexes to be vacuumed or cleaned and
indexes_processed as the number of indexes vacuumed or cleaned up so
far.
Author: Sami Imseih, based on suggestions by Nathan Bossart, Peter Geoghegan and Masahiko Sawada
Reviewed by: Nathan Bossart, Masahiko Sawada
Discussion: https://www.postgresql.org/message-id/flat/[email protected]
---
doc/src/sgml/monitoring.sgml | 25 ++++++++++++++++++
src/backend/access/heap/vacuumlazy.c | 19 ++++++++++++++
src/backend/catalog/system_views.sql | 3 ++-
src/backend/commands/vacuumparallel.c | 37 +++++++++++++++++++++++++++
src/include/commands/progress.h | 2 ++
src/test/regress/expected/rules.out | 4 ++-
6 files changed, 88 insertions(+), 2 deletions(-)
diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml
index 56d9b37..0599384 100644
--- a/doc/src/sgml/monitoring.sgml
+++ b/doc/src/sgml/monitoring.sgml
@@ -6396,6 +6396,31 @@ SELECT pg_stat_get_backend_pid(s.backendid) AS pid,
Number of dead tuples collected since the last index vacuum cycle.
</para></entry>
</row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>indexes_total</structfield> <type>bigint</type>
+ </para>
+ <para>
+ The number of indexes to be processed in the
+ <literal>vacuuming indexes</literal>
+ or <literal>cleaning up indexes</literal> phase. It is set to
+ <literal>0</literal> when there are no indexes to process
+ or when failsafe is triggered during the vacuum.
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>indexes_completed</structfield> <type>bigint</type>
+ </para>
+ <para>
+ The number of indexes already processed in the
+ <literal>vacuuming indexes</literal>
+ or <literal>cleaning up indexes</literal> phase. It is set to
+ <literal>0</literal> when vacuum is not in any of these phases.
+ </para></entry>
+ </row>
</tbody>
</tgroup>
</table>
diff --git a/src/backend/access/heap/vacuumlazy.c b/src/backend/access/heap/vacuumlazy.c
index b802ed2..f01341e 100644
--- a/src/backend/access/heap/vacuumlazy.c
+++ b/src/backend/access/heap/vacuumlazy.c
@@ -418,6 +418,9 @@ heap_vacuum_rel(Relation rel, VacuumParams *params,
vacrel->rel = rel;
vac_open_indexes(vacrel->rel, RowExclusiveLock, &vacrel->nindexes,
&vacrel->indrels);
+ /* Report the number of indexes to vacuum/cleanup */
+ pgstat_progress_update_param(PROGRESS_VACUUM_INDEXES_TOTAL, vacrel->nindexes);
+
if (instrument && vacrel->nindexes > 0)
{
/* Copy index names used by instrumentation (not error reporting) */
@@ -2327,6 +2330,8 @@ lazy_vacuum_all_indexes(LVRelState *vacrel)
vacrel->indstats[idx] =
lazy_vacuum_one_index(indrel, istat, vacrel->old_live_tuples,
vacrel);
+ /* Report the number of indexes vacuumed */
+ pgstat_progress_update_param(PROGRESS_VACUUM_INDEXES_COMPLETED, idx + 1);
if (lazy_check_wraparound_failsafe(vacrel))
{
@@ -2361,6 +2366,10 @@ lazy_vacuum_all_indexes(LVRelState *vacrel)
vacrel->dead_items->num_items == vacrel->lpdead_items);
Assert(allindexes || vacrel->failsafe_active);
+ /* Report that we're done vacuuming indexes */
+ pgstat_progress_update_param(PROGRESS_VACUUM_INDEXES_TOTAL, 0);
+ pgstat_progress_update_param(PROGRESS_VACUUM_INDEXES_COMPLETED, 0);
+
/*
* Increase and report the number of index scans.
*
@@ -2625,6 +2634,10 @@ lazy_check_wraparound_failsafe(LVRelState *vacrel)
vacrel->do_index_cleanup = false;
vacrel->do_rel_truncate = false;
+ /* Report that we're no longer vacuuming indexes */
+ pgstat_progress_update_param(PROGRESS_VACUUM_INDEXES_TOTAL, 0);
+ pgstat_progress_update_param(PROGRESS_VACUUM_INDEXES_COMPLETED, 0);
+
ereport(WARNING,
(errmsg("bypassing nonessential maintenance of table \"%s.%s.%s\" as a failsafe after %d index scans",
get_database_name(MyDatabaseId),
@@ -2671,6 +2684,8 @@ lazy_cleanup_all_indexes(LVRelState *vacrel)
vacrel->indstats[idx] =
lazy_cleanup_one_index(indrel, istat, reltuples,
estimated_count, vacrel);
+ /* Report the number of indexes cleaned */
+ pgstat_progress_update_param(PROGRESS_VACUUM_INDEXES_COMPLETED, idx + 1);
}
}
else
@@ -2680,6 +2695,10 @@ lazy_cleanup_all_indexes(LVRelState *vacrel)
vacrel->num_index_scans,
estimated_count);
}
+
+ /* Report that we're done cleaning up indexes */
+ pgstat_progress_update_param(PROGRESS_VACUUM_INDEXES_TOTAL, 0);
+ pgstat_progress_update_param(PROGRESS_VACUUM_INDEXES_COMPLETED, 0);
}
/*
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fedaed5..0c8261b 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1163,7 +1163,8 @@ CREATE VIEW pg_stat_progress_vacuum AS
END AS phase,
S.param2 AS heap_blks_total, S.param3 AS heap_blks_scanned,
S.param4 AS heap_blks_vacuumed, S.param5 AS index_vacuum_count,
- S.param6 AS max_dead_tuples, S.param7 AS num_dead_tuples
+ S.param6 AS max_dead_tuples, S.param7 AS num_dead_tuples,
+ S.param8 AS indexes_total, S.param9 AS indexes_completed
FROM pg_stat_get_progress_info('VACUUM') AS S
LEFT JOIN pg_database D ON S.datid = D.oid;
diff --git a/src/backend/commands/vacuumparallel.c b/src/backend/commands/vacuumparallel.c
index 1753da6..ad61a54 100644
--- a/src/backend/commands/vacuumparallel.c
+++ b/src/backend/commands/vacuumparallel.c
@@ -30,6 +30,7 @@
#include "access/table.h"
#include "access/xact.h"
#include "catalog/index.h"
+#include "commands/progress.h"
#include "commands/vacuum.h"
#include "optimizer/paths.h"
#include "pgstat.h"
@@ -213,6 +214,7 @@ static void parallel_vacuum_process_one_index(ParallelVacuumState *pvs, Relation
static bool parallel_vacuum_index_is_parallel_safe(Relation indrel, int num_index_scans,
bool vacuum);
static void parallel_vacuum_error_callback(void *arg);
+static void parallel_vacuum_progress_callback(void *arg);
/*
* Try to enter parallel mode and create a parallel context. Then initialize
@@ -288,6 +290,10 @@ parallel_vacuum_init(Relation rel, Relation *indrels, int nindexes,
shm_toc_estimate_chunk(&pcxt->estimator, est_dead_items_len);
shm_toc_estimate_keys(&pcxt->estimator, 1);
+ /* Setup the Parallel Progress Callback */
+ pvs->pcxt->parallel_progress_callback = parallel_vacuum_progress_callback;
+ pvs->pcxt->parallel_progress_callback_arg = pvs;
+
/*
* Estimate space for BufferUsage and WalUsage --
* PARALLEL_VACUUM_KEY_BUFFER_USAGE and PARALLEL_VACUUM_KEY_WAL_USAGE.
@@ -885,6 +891,13 @@ parallel_vacuum_process_one_index(ParallelVacuumState *pvs, Relation indrel,
/* Reset error traceback information */
pvs->status = PARALLEL_INDVAC_STATUS_COMPLETED;
+
+ /*
+ * If we are the leader, update index vacuum progress.
+ */
+ if (!IsParallelWorker())
+ pvs->pcxt->parallel_progress_callback(pvs->pcxt->parallel_progress_callback_arg);
+
pfree(pvs->indname);
pvs->indname = NULL;
}
@@ -1071,3 +1084,27 @@ parallel_vacuum_error_callback(void *arg)
return;
}
}
+
+/*
+ * Callback to report index vacuum progress.
+ * Vacuum Progress should only be reported
+ * to the leader process.
+ */
+static void
+parallel_vacuum_progress_callback(void *arg)
+{
+ ParallelVacuumState *pvs = (ParallelVacuumState *)arg;
+ int indexes_completed = 0;
+
+ Assert(!IsParallelWorker());
+
+ for (int i = 0; i < pvs->nindexes; i++)
+ {
+ PVIndStats *indstats = &(pvs->indstats[i]);
+
+ if (indstats->status == PARALLEL_INDVAC_STATUS_COMPLETED)
+ indexes_completed++;
+ }
+
+ pgstat_progress_update_param(PROGRESS_VACUUM_INDEXES_COMPLETED, indexes_completed);
+}
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index a28938c..0d1724e 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -25,6 +25,8 @@
#define PROGRESS_VACUUM_NUM_INDEX_VACUUMS 4
#define PROGRESS_VACUUM_MAX_DEAD_TUPLES 5
#define PROGRESS_VACUUM_NUM_DEAD_TUPLES 6
+#define PROGRESS_VACUUM_INDEXES_TOTAL 7
+#define PROGRESS_VACUUM_INDEXES_COMPLETED 8
/* Phases of vacuum (as advertised via PROGRESS_VACUUM_PHASE) */
#define PROGRESS_VACUUM_PHASE_SCAN_HEAP 1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index fc3cde3..9c3442f 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -2017,7 +2017,9 @@ pg_stat_progress_vacuum| SELECT s.pid,
s.param4 AS heap_blks_vacuumed,
s.param5 AS index_vacuum_count,
s.param6 AS max_dead_tuples,
- s.param7 AS num_dead_tuples
+ s.param7 AS num_dead_tuples,
+ s.param8 AS indexes_total,
+ s.param9 AS indexes_completed
FROM (pg_stat_get_progress_info('VACUUM'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
LEFT JOIN pg_database d ON ((s.datid = d.oid)));
pg_stat_recovery_prefetch| SELECT s.stats_reset,
--
2.32.1 (Apple Git-133)
^ permalink raw reply [nested|flat] 44+ messages in thread
* Re: Add index scan progress to pg_stat_progress_vacuum
@ 2022-05-26 15:43 Masahiko Sawada <[email protected]>
parent: Imseih (AWS), Sami <[email protected]>
1 sibling, 2 replies; 44+ messages in thread
From: Masahiko Sawada @ 2022-05-26 15:43 UTC (permalink / raw)
To: Imseih (AWS), Sami <[email protected]>; +Cc: Robert Haas <[email protected]>; Bossart, Nathan <[email protected]>; Andres Freund <[email protected]>; Peter Geoghegan <[email protected]>; Justin Pryzby <[email protected]>; pgsql-hackers
On Fri, May 6, 2022 at 4:26 AM Imseih (AWS), Sami <[email protected]> wrote:
>
> Thank you for the feedback!
>
> > I think we can pass the progress update function to
> > WaitForParallelWorkersToFinish(), which seems simpler. And we can call
>
> Directly passing the callback to WaitForParallelWorkersToFinish
> will require us to modify the function signature.
>
> To me, it seemed simpler and touches less code to have
> the caller set the callback in the ParallelContext.
Okay, but if we do that, I think we should add comments about when
it's used. The callback is used only when
WaitForParallelWorkersToFinish(), but not when
WaitForParallelWorkersToExit().
Another idea I came up with is that we can wait for all index vacuums
to finish while checking and updating the progress information, and
then calls WaitForParallelWorkersToFinish after confirming all index
status became COMPLETED. That way, we don’t need to change the
parallel query infrastructure. What do you think?
>
> > the function after updating the index status to
> > PARALLEL_INDVAC_STATUS_COMPLETED.
>
> I also like this better. Will make the change.
>
> > BTW, currently we don't need a lock for touching index status since
> > each worker touches different indexes. But after this patch, the
> > leader will touch all index status, do we need a lock for that?
>
> I do not think locking is needed here. The leader and workers
> will continue to touch different indexes to update the status.
>
> However, if the process is a leader, it will call the function
> which will go through indstats and count how many
> Indexes have a status of PARALLEL_INDVAC_STATUS_COMPLETED.
> This value is then reported to the leaders backend only.
I was concerned that the leader process could report the wrong
progress if updating and checking index status happen concurrently.
But I think it should be fine since we can read PVIndVacStatus
atomically.
Regards,
--
Masahiko Sawada
EDB: https://www.enterprisedb.com/
^ permalink raw reply [nested|flat] 44+ messages in thread
* Re: Add index scan progress to pg_stat_progress_vacuum
@ 2022-05-27 01:52 Imseih (AWS), Sami <[email protected]>
parent: Masahiko Sawada <[email protected]>
1 sibling, 1 reply; 44+ messages in thread
From: Imseih (AWS), Sami @ 2022-05-27 01:52 UTC (permalink / raw)
To: Masahiko Sawada <[email protected]>; +Cc: Robert Haas <[email protected]>; Bossart, Nathan <[email protected]>; Andres Freund <[email protected]>; Peter Geoghegan <[email protected]>; Justin Pryzby <[email protected]>; pgsql-hackers
> Another idea I came up with is that we can wait for all index vacuums
> to finish while checking and updating the progress information, and
> then calls WaitForParallelWorkersToFinish after confirming all index
> status became COMPLETED. That way, we don’t need to change the
> parallel query infrastructure. What do you think?
Thinking about this a bit more, the idea of using
WaitForParallelWorkersToFinish
Will not work if you have a leader worker that is
stuck on a large index. The progress will not be updated
until the leader completes. Even if the parallel workers
finish.
What are your thought about piggybacking on the
vacuum_delay_point to update progress. The leader can
perhaps keep a counter to update progress every few thousand
calls to vacuum_delay_point.
This goes back to your original idea to keep updating progress
while scanning the indexes.
/*
* vacuum_delay_point --- check for interrupts and cost-based delay.
*
* This should be called in each major loop of VACUUM processing,
* typically once per page processed.
*/
void
vacuum_delay_point(void)
{
---
Sami Imseih
Amazon Web Services
^ permalink raw reply [nested|flat] 44+ messages in thread
* Re: Add index scan progress to pg_stat_progress_vacuum
@ 2022-06-03 05:39 Masahiko Sawada <[email protected]>
parent: Imseih (AWS), Sami <[email protected]>
0 siblings, 0 replies; 44+ messages in thread
From: Masahiko Sawada @ 2022-06-03 05:39 UTC (permalink / raw)
To: Imseih (AWS), Sami <[email protected]>; +Cc: Robert Haas <[email protected]>; Bossart, Nathan <[email protected]>; Andres Freund <[email protected]>; Peter Geoghegan <[email protected]>; Justin Pryzby <[email protected]>; pgsql-hackers
On Fri, May 27, 2022 at 10:52 AM Imseih (AWS), Sami <[email protected]> wrote:
>
> > Another idea I came up with is that we can wait for all index vacuums
> > to finish while checking and updating the progress information, and
> > then calls WaitForParallelWorkersToFinish after confirming all index
> > status became COMPLETED. That way, we don’t need to change the
> > parallel query infrastructure. What do you think?
>
> Thinking about this a bit more, the idea of using
> WaitForParallelWorkersToFinish
> Will not work if you have a leader worker that is
> stuck on a large index. The progress will not be updated
> until the leader completes. Even if the parallel workers
> finish.
Right.
>
> What are your thought about piggybacking on the
> vacuum_delay_point to update progress. The leader can
> perhaps keep a counter to update progress every few thousand
> calls to vacuum_delay_point.
>
> This goes back to your original idea to keep updating progress
> while scanning the indexes.
I think we can have the leader process wait for all index statuses to
become COMPLETED before WaitForParallelWorkersToFinish(). While
waiting for it, the leader can update its progress information. After
the leader confirmed all index statuses became COMPLETED, it can wait
for the workers to finish by WaitForParallelWorkersToFinish().
Regarding waiting in vacuum_delay_point, it might be a side effect as
it’s called every page and used not only by vacuum such as analyze,
but it seems to be worth trying.
Regards,
--
Masahiko Sawada
EDB: https://www.enterprisedb.com/
^ permalink raw reply [nested|flat] 44+ messages in thread
* Re: Add index scan progress to pg_stat_progress_vacuum
@ 2022-06-06 14:41 Robert Haas <[email protected]>
parent: Masahiko Sawada <[email protected]>
1 sibling, 1 reply; 44+ messages in thread
From: Robert Haas @ 2022-06-06 14:41 UTC (permalink / raw)
To: Masahiko Sawada <[email protected]>; +Cc: Imseih (AWS), Sami <[email protected]>; Bossart, Nathan <[email protected]>; Andres Freund <[email protected]>; Peter Geoghegan <[email protected]>; Justin Pryzby <[email protected]>; pgsql-hackers
On Thu, May 26, 2022 at 11:43 AM Masahiko Sawada <[email protected]> wrote:
> Another idea I came up with is that we can wait for all index vacuums
> to finish while checking and updating the progress information, and
> then calls WaitForParallelWorkersToFinish after confirming all index
> status became COMPLETED. That way, we don’t need to change the
> parallel query infrastructure. What do you think?
+1 from me. It doesn't seem to me that we should need to add something
like parallel_vacuum_progress_callback in order to solve this problem,
because the parallel index vacuum code could just do the waiting
itself, as you propose here.
The question Sami asks him his reply is a good one, though -- who is
to say that the leader only needs to update progress at the end, once
it's finished the index it's handling locally? There will need to be a
callback system of some kind to allow the leader to update progress as
other workers finish, even if the leader is still working. I am not
too sure that the idea of using the vacuum delay points is the best
plan. I think we should try to avoid piggybacking on such general
infrastructure if we can, and instead look for a way to tie this to
something that is specific to parallel vacuum. However, I haven't
studied the problem so I'm not sure whether there's a reasonable way
to do that.
--
Robert Haas
EDB: http://www.enterprisedb.com
^ permalink raw reply [nested|flat] 44+ messages in thread
* Re: Add index scan progress to pg_stat_progress_vacuum
@ 2022-06-20 06:35 Masahiko Sawada <[email protected]>
parent: Robert Haas <[email protected]>
0 siblings, 2 replies; 44+ messages in thread
From: Masahiko Sawada @ 2022-06-20 06:35 UTC (permalink / raw)
To: Robert Haas <[email protected]>; +Cc: Imseih (AWS), Sami <[email protected]>; Bossart, Nathan <[email protected]>; Andres Freund <[email protected]>; Peter Geoghegan <[email protected]>; Justin Pryzby <[email protected]>; pgsql-hackers
On Mon, Jun 6, 2022 at 11:42 PM Robert Haas <[email protected]> wrote:
>
> On Thu, May 26, 2022 at 11:43 AM Masahiko Sawada <[email protected]> wrote:
> > Another idea I came up with is that we can wait for all index vacuums
> > to finish while checking and updating the progress information, and
> > then calls WaitForParallelWorkersToFinish after confirming all index
> > status became COMPLETED. That way, we don’t need to change the
> > parallel query infrastructure. What do you think?
>
> +1 from me. It doesn't seem to me that we should need to add something
> like parallel_vacuum_progress_callback in order to solve this problem,
> because the parallel index vacuum code could just do the waiting
> itself, as you propose here.
>
> The question Sami asks him his reply is a good one, though -- who is
> to say that the leader only needs to update progress at the end, once
> it's finished the index it's handling locally? There will need to be a
> callback system of some kind to allow the leader to update progress as
> other workers finish, even if the leader is still working. I am not
> too sure that the idea of using the vacuum delay points is the best
> plan. I think we should try to avoid piggybacking on such general
> infrastructure if we can, and instead look for a way to tie this to
> something that is specific to parallel vacuum. However, I haven't
> studied the problem so I'm not sure whether there's a reasonable way
> to do that.
One idea would be to add a flag, say report_parallel_vacuum_progress,
to IndexVacuumInfo struct and expect index AM to check and update the
parallel index vacuum progress, say every 1GB blocks processed. The
flag is true only when the leader process is vacuuming an index.
Regards,
--
Masahiko Sawada
EDB: https://www.enterprisedb.com/
^ permalink raw reply [nested|flat] 44+ messages in thread
* Re: Add index scan progress to pg_stat_progress_vacuum
@ 2022-08-02 18:06 Jacob Champion <[email protected]>
parent: Masahiko Sawada <[email protected]>
1 sibling, 0 replies; 44+ messages in thread
From: Jacob Champion @ 2022-08-02 18:06 UTC (permalink / raw)
To: Masahiko Sawada <[email protected]>; Robert Haas <[email protected]>; +Cc: Imseih (AWS), Sami <[email protected]>; Bossart, Nathan <[email protected]>; Andres Freund <[email protected]>; Peter Geoghegan <[email protected]>; Justin Pryzby <[email protected]>; pgsql-hackers
This entry has been waiting on author input for a while (our current
threshold is roughly two weeks), so I've marked it Returned with
Feedback.
Once you think the patchset is ready for review again, you (or any
interested party) can resurrect the patch entry by visiting
https://commitfest.postgresql.org/38/3617/
and changing the status to "Needs Review", and then changing the
status again to "Move to next CF". (Don't forget the second step;
hopefully we will have streamlined this in the near future!)
Thanks,
--Jacob
^ permalink raw reply [nested|flat] 44+ messages in thread
* Re: Add index scan progress to pg_stat_progress_vacuum
@ 2022-10-10 16:40 Imseih (AWS), Sami <[email protected]>
0 siblings, 0 replies; 44+ messages in thread
From: Imseih (AWS), Sami @ 2022-10-10 16:40 UTC (permalink / raw)
To: pgsql-hackers; +Cc: Masahiko Sawada <[email protected]>; Robert Haas <[email protected]>
> One idea would be to add a flag, say report_parallel_vacuum_progress,
> to IndexVacuumInfo struct and expect index AM to check and update the
> parallel index vacuum progress, say every 1GB blocks processed. The
> flag is true only when the leader process is vacuuming an index.
Sorry for the long delay on this. I have taken the approach as suggested
by Sawada-san and Robert and attached is v12.
1. The patch introduces a new counter in the shared memory already
used by the parallel leader and workers to keep track of the number
of indexes completed. This way there is no reason to loop through
the index status everytime we want to get the status of indexes completed.
2. A new function in vacuumparallel.c will be used to update
the progress of a indexes completed by reading from the
counter created in point #1.
3. The function is called during the vacuum_delay_point as a
matter of convenience, since it's called in all major vacuum
loops. The function will only do anything if the caller
sets a boolean to report progress. Doing so will also ensure
progress is being reported in case the parallel workers completed
before the leader.
4. Rather than adding any complexity to WaitForParallelWorkersToFinish
and introducing a new callback, vacuumparallel.c will wait until
the number of vacuum workers is 0 and then process to call
WaitForParallelWorkersToFinish as it does.
5. Went back to the idea of adding a new view called pg_stat_progress_vacuum_index
which is accomplished by adding a new type called VACUUM_PARALLEL in progress.h
Thanks,
Sami Imseih
Amazon Web Servies (AWS)
Attachments:
[application/octet-stream] v12-0001--Show-progress-for-index-vacuums.patch (23.0K, ../../[email protected]/3-v12-0001--Show-progress-for-index-vacuums.patch)
download | inline diff:
From fd394f0bf01406f850206a6c4a81ff187a685a69 Mon Sep 17 00:00:00 2001
From: "Imseih (AWS)" <[email protected]>
Date: Mon, 10 Oct 2022 11:22:25 -0500
Subject: [PATCH v12 1/1] Add 2 new columns to pg_stat_progress_vacuum. The
columns are indexes_total as the total indexes to be vacuumed or cleaned and
indexes_processed as the number of indexes vacuumed or cleaned up so far.
Also, introduce a new view called pg_stat_progress_vacuum_index that
exposes the current index being vacuumed.
Author: Sami Imseih, based on suggestions by Nathan Bossart, Peter Geoghegan and Masahiko Sawada
Reviewed by: Nathan Bossart, Masahiko Sawada
Discussion: https://www.postgresql.org/message-id/flat/[email protected]
---
doc/src/sgml/monitoring.sgml | 92 ++++++++++++++++++++++
doc/src/sgml/ref/vacuum.sgml | 8 +-
src/backend/access/heap/vacuumlazy.c | 44 ++++++++++-
src/backend/catalog/system_views.sql | 20 ++++-
src/backend/commands/vacuum.c | 6 ++
src/backend/commands/vacuumparallel.c | 106 +++++++++++++++++++++++++-
src/backend/utils/adt/pgstatfuncs.c | 2 +
src/include/commands/progress.h | 4 +
src/include/commands/vacuum.h | 2 +
src/include/utils/backend_progress.h | 7 +-
src/test/regress/expected/rules.out | 17 ++++-
11 files changed, 301 insertions(+), 7 deletions(-)
diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml
index 342b20ebeb..473c76e6e8 100644
--- a/doc/src/sgml/monitoring.sgml
+++ b/doc/src/sgml/monitoring.sgml
@@ -392,6 +392,15 @@ postgres 27093 0.0 0.0 30096 2752 ? Ss 11:34 0:00 postgres: ser
</entry>
</row>
+ <row>
+ <entry><structname>pg_stat_progress_vacuum_index</structname><indexterm><primary>pg_stat_progress_vacuum_index</primary>
+ </indexterm>
+ </entry>
+ <entry>One row for each backend (including autovacuum worker processes) performing the <literal>vacuuming indexes</literal>
+ or <literal>cleaning up indexes</literal> phase of a <command>VACUUM</command>.
+ </entry>
+ </row>
+
<row>
<entry><structname>pg_stat_progress_cluster</structname><indexterm><primary>pg_stat_progress_cluster</primary></indexterm></entry>
<entry>One row for each backend running
@@ -6414,6 +6423,89 @@ FROM pg_stat_get_backend_idset() AS backendid;
Number of dead tuples collected since the last index vacuum cycle.
</para></entry>
</row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>indexes_total</structfield> <type>bigint</type>
+ </para>
+ <para>
+ Number of indexes that wil be vacuumed. This value will be
+ <literal>0</literal> if there are no indexes to vacuum or
+ vacuum failsafe is triggered. See <xref linkend="guc-vacuum-failsafe-age"/>
+ for more on vacuum failsafe.
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>indexes_completed</structfield> <type>bigint</type>
+ </para>
+ <para>
+ Number of indexes vacuumed in the current vacuum cycle.
+ </para></entry>
+ </row>
+ </tbody>
+ </tgroup>
+ </table>
+
+ <indexterm>
+ <primary>pg_stat_progress_vacuum_index</primary>
+ </indexterm>
+
+ <para>
+ Whenever <command>VACUUM</command> is running, the
+ <structname>pg_stat_progress_vacuum_index</structname> view will contain
+ one row for each backend (including autovacuum worker processes) performing
+ the <literal>vacuuming indexes</literal> or <literal>cleaning up indexes</literal>
+ phase of a <command>VACUUM</command>.
+ </para>
+
+ <table id="pg-stat-progress-vacuum-view_index" xreflabel="pg_stat_progress_vacuum_index">
+ <title><structname>pg_stat_progress_vacuum_index</structname> View</title>
+ <tgroup cols="1">
+ <thead>
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ Column Type
+ </para>
+ <para>
+ Description
+ </para></entry>
+ </row>
+ </thead>
+
+ <tbody>
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>pid</structfield> <type>integer</type>
+ </para>
+ <para>
+ Process ID of backend.
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>leader_pid</structfield> <type>integer</type>
+ </para>
+ <para>
+ Process ID of the leader backend in a parallel <command>VACUUM</command>. This value
+ will match the <structfield>pid</structfield> value whenever the leader
+ is processing an index or the <command>VACUUM</command> is not using parallel.
+ This field can be joined to <structfield>pid</structfield>
+ of <structfield>pg_stat_progress_vacuum</structfield> to get more details about
+ the <command>VACUUM</command>.
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>indexrelid</structfield> <type>integer</type>
+ </para>
+ <para>
+ OID of the index being processed in the current vacuum phase.
+ </para></entry>
+ </row>
</tbody>
</tgroup>
</table>
diff --git a/doc/src/sgml/ref/vacuum.sgml b/doc/src/sgml/ref/vacuum.sgml
index c582021d29..0c08d9ac6d 100644
--- a/doc/src/sgml/ref/vacuum.sgml
+++ b/doc/src/sgml/ref/vacuum.sgml
@@ -411,7 +411,13 @@ VACUUM [ FULL ] [ FREEZE ] [ VERBOSE ] [ ANALYZE ] [ <replaceable class="paramet
<para>
Each backend running <command>VACUUM</command> without the
<literal>FULL</literal> option will report its progress in the
- <structname>pg_stat_progress_vacuum</structname> view. Backends running
+ <structname>pg_stat_progress_vacuum</structname> view.
+ <structname>pg_stat_progress_vacuum</structname> view. Whenever a
+ <command>VACUUM</command> is in the <literal>vacuuming indexes</literal>
+ or <literal>cleaning up indexes</literal> phase,
+ the current index being processed is reported in
+ <structname>pg_stat_progress_vacuum_index</structname>.
+ <command>VACUUM FULL</command> will report their progress in the Backends running
<command>VACUUM FULL</command> will instead report their progress in the
<structname>pg_stat_progress_cluster</structname> view. See
<xref linkend="vacuum-progress-reporting"/> and
diff --git a/src/backend/access/heap/vacuumlazy.c b/src/backend/access/heap/vacuumlazy.c
index dfbe37472f..b057d18dd9 100644
--- a/src/backend/access/heap/vacuumlazy.c
+++ b/src/backend/access/heap/vacuumlazy.c
@@ -350,8 +350,10 @@ heap_vacuum_rel(Relation rel, VacuumParams *params,
}
}
+ /* start the vacuum progress command and report the leader pid. */
pgstat_progress_start_command(PROGRESS_COMMAND_VACUUM,
RelationGetRelid(rel));
+ pgstat_progress_update_param(PROGRESS_VACUUM_LEADER_PID, MyProcPid);
/*
* Get OldestXmin cutoff, which is used to determine which deleted tuples
@@ -420,6 +422,10 @@ heap_vacuum_rel(Relation rel, VacuumParams *params,
vacrel->rel = rel;
vac_open_indexes(vacrel->rel, RowExclusiveLock, &vacrel->nindexes,
&vacrel->indrels);
+
+ /* report number of indexes to vacuum */
+ pgstat_progress_update_param(PROGRESS_VACUUM_INDEX_TOTAL, vacrel->nindexes);
+
if (instrument && vacrel->nindexes > 0)
{
/* Copy index names used by instrumentation (not error reporting) */
@@ -2337,10 +2343,21 @@ lazy_vacuum_all_indexes(LVRelState *vacrel)
Relation indrel = vacrel->indrels[idx];
IndexBulkDeleteResult *istat = vacrel->indstats[idx];
+ /* report the index relid being vacuumed */
+ pgstat_progress_update_param(PROGRESS_VACUUM_INDRELID, RelationGetRelid(indrel));
+
vacrel->indstats[idx] =
lazy_vacuum_one_index(indrel, istat, vacrel->old_live_tuples,
vacrel);
+ /*
+ * Done vacuuming an index.
+ * Increment the indexes completed and reset the index relid to 0
+ */
+ pgstat_progress_update_param(PROGRESS_VACUUM_INDEX_COMPLETED,
+ idx + 1);
+ pgstat_progress_update_param(PROGRESS_VACUUM_INDRELID, 0);
+
if (lazy_check_wraparound_failsafe(vacrel))
{
/* Wraparound emergency -- end current index scan */
@@ -2384,6 +2401,13 @@ lazy_vacuum_all_indexes(LVRelState *vacrel)
pgstat_progress_update_param(PROGRESS_VACUUM_NUM_INDEX_VACUUMS,
vacrel->num_index_scans);
+ /*
+ * Reset the indexes completed at this point.
+ * If we end up in another index vacuum cycle, we will
+ * start counting from the start.
+ */
+ pgstat_progress_update_param(PROGRESS_VACUUM_INDEX_COMPLETED, 0);
+
return allindexes;
}
@@ -2633,10 +2657,17 @@ lazy_check_wraparound_failsafe(LVRelState *vacrel)
{
vacrel->failsafe_active = true;
- /* Disable index vacuuming, index cleanup, and heap rel truncation */
+ /*
+ * Disable index vacuuming, index cleanup, and heap rel truncation
+ *
+ * Also, report to progress.h that we are no longer tracking
+ * index vacuum/cleanup.
+ */
vacrel->do_index_vacuuming = false;
vacrel->do_index_cleanup = false;
vacrel->do_rel_truncate = false;
+ pgstat_progress_update_param(PROGRESS_VACUUM_INDEX_TOTAL, 0);
+ pgstat_progress_update_param(PROGRESS_VACUUM_INDEX_COMPLETED, 0);
ereport(WARNING,
(errmsg("bypassing nonessential maintenance of table \"%s.%s.%s\" as a failsafe after %d index scans",
@@ -2681,9 +2712,20 @@ lazy_cleanup_all_indexes(LVRelState *vacrel)
Relation indrel = vacrel->indrels[idx];
IndexBulkDeleteResult *istat = vacrel->indstats[idx];
+ /*+ report the index relid being cleaned */
+ pgstat_progress_update_param(PROGRESS_VACUUM_INDRELID, RelationGetRelid(indrel));
+
vacrel->indstats[idx] =
lazy_cleanup_one_index(indrel, istat, reltuples,
estimated_count, vacrel);
+
+ /*
+ * Done cleaning an index.
+ * Increment the indexes completed and reset the index relid to 0
+ */
+ pgstat_progress_update_param(PROGRESS_VACUUM_INDEX_COMPLETED,
+ idx + 1);
+ pgstat_progress_update_param(PROGRESS_VACUUM_INDRELID, 0);
}
}
else
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index 55f7ec79e0..5b5e7b4080 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1162,10 +1162,28 @@ CREATE VIEW pg_stat_progress_vacuum AS
END AS phase,
S.param2 AS heap_blks_total, S.param3 AS heap_blks_scanned,
S.param4 AS heap_blks_vacuumed, S.param5 AS index_vacuum_count,
- S.param6 AS max_dead_tuples, S.param7 AS num_dead_tuples
+ S.param6 AS max_dead_tuples, S.param7 AS num_dead_tuples,
+ S.param10 AS indexes_total, S.param11 AS indexes_completed
FROM pg_stat_get_progress_info('VACUUM') AS S
LEFT JOIN pg_database D ON S.datid = D.oid;
+CREATE VIEW pg_stat_progress_vacuum_index AS
+ SELECT
+ S.pid AS pid,
+ S.param9 AS leader_pid,
+ S.param8 AS indexrelid
+ FROM pg_stat_get_progress_info('VACUUM') AS S
+ LEFT JOIN pg_database D ON S.datid = D.oid
+ WHERE S.param1 in (2, 4) AND S.param8 > 0
+ UNION ALL
+ SELECT
+ S.pid AS pid,
+ S.param9 AS leader_pid,
+ S.param8 AS indexrelid
+ FROM pg_stat_get_progress_info('VACUUM_PARALLEL') AS S
+ LEFT JOIN pg_database D ON S.datid = D.oid
+ WHERE S.param1 in (2, 4) AND S.param8 > 0;
+
CREATE VIEW pg_stat_progress_cluster AS
SELECT
S.pid AS pid,
diff --git a/src/backend/commands/vacuum.c b/src/backend/commands/vacuum.c
index 7ccde07de9..19a2692704 100644
--- a/src/backend/commands/vacuum.c
+++ b/src/backend/commands/vacuum.c
@@ -2173,12 +2173,18 @@ vac_close_indexes(int nindexes, Relation *Irel, LOCKMODE lockmode)
*
* This should be called in each major loop of VACUUM processing,
* typically once per page processed.
+ *
+ * NOTE: For convenience, parallel_vacuum_progress_report, is called
+ * here so the leader can report the number of indexes vacuumed in
+ * while inside all the major VACUUM loops.
*/
void
vacuum_delay_point(void)
{
double msec = 0;
+ parallel_vacuum_progress_report();
+
/* Always check for interrupts */
CHECK_FOR_INTERRUPTS();
diff --git a/src/backend/commands/vacuumparallel.c b/src/backend/commands/vacuumparallel.c
index f26d796e52..65a71cbac9 100644
--- a/src/backend/commands/vacuumparallel.c
+++ b/src/backend/commands/vacuumparallel.c
@@ -30,6 +30,7 @@
#include "access/table.h"
#include "access/xact.h"
#include "catalog/index.h"
+#include "commands/progress.h"
#include "commands/vacuum.h"
#include "optimizer/paths.h"
#include "pgstat.h"
@@ -103,6 +104,20 @@ typedef struct PVShared
/* Counter for vacuuming and cleanup */
pg_atomic_uint32 idx;
+
+ /*
+ * Counter for vacuuming and cleanup progress reporting.
+ * This value is used to report index vacuum/cleanup progress
+ * in parallel_vacuum_progress_report. We keep this
+ * counter to avoid having to loop through
+ * ParallelVacuumState->indstats to determine the number
+ * of indexes completed.
+ */
+ pg_atomic_uint32 idx_completed_progress;
+
+ /* track the leader pid of a parallel vacuum */
+ int leader_pid;
+
} PVShared;
/* Status used during parallel index vacuum or cleanup */
@@ -214,6 +229,9 @@ static bool parallel_vacuum_index_is_parallel_safe(Relation indrel, int num_inde
bool vacuum);
static void parallel_vacuum_error_callback(void *arg);
+static pg_atomic_uint32 *index_vacuum_completed = NULL;
+static bool report_parallel_vacuum_progress = false;
+
/*
* Try to enter parallel mode and create a parallel context. Then initialize
* shared memory state.
@@ -364,6 +382,9 @@ parallel_vacuum_init(Relation rel, Relation *indrels, int nindexes,
pg_atomic_init_u32(&(shared->cost_balance), 0);
pg_atomic_init_u32(&(shared->active_nworkers), 0);
pg_atomic_init_u32(&(shared->idx), 0);
+ pg_atomic_init_u32(&(shared->idx_completed_progress), 0);
+
+ shared->leader_pid = MyProcPid;
shm_toc_insert(pcxt->toc, PARALLEL_VACUUM_KEY_SHARED, shared);
pvs->shared = shared;
@@ -618,8 +639,9 @@ parallel_vacuum_process_all_indexes(ParallelVacuumState *pvs, int num_index_scan
vacuum));
}
- /* Reset the parallel index processing counter */
+ /* Reset the parallel index processing counter ( index proress counter also ) */
pg_atomic_write_u32(&(pvs->shared->idx), 0);
+ pg_atomic_write_u32(&(pvs->shared->idx_completed_progress), 0);
/* Setup the shared cost-based vacuum delay and launch workers */
if (nworkers > 0)
@@ -657,6 +679,13 @@ parallel_vacuum_process_all_indexes(ParallelVacuumState *pvs, int num_index_scan
/* Enable shared cost balance for leader backend */
VacuumSharedCostBalance = &(pvs->shared->cost_balance);
VacuumActiveNWorkers = &(pvs->shared->active_nworkers);
+
+ /*
+ * If we are launching a parallel vacuum/cleanup,
+ * setup the tracking.
+ */
+ report_parallel_vacuum_progress = true;
+ index_vacuum_completed = &(pvs->shared->idx_completed_progress);
}
if (vacuum)
@@ -682,13 +711,36 @@ parallel_vacuum_process_all_indexes(ParallelVacuumState *pvs, int num_index_scan
*/
parallel_vacuum_process_safe_indexes(pvs);
+ /*
+ * In case the leader completes vacuuming all
+ * its indexes before the parallel workers do,
+ * it can spin here waiting for the number of
+ * active workers to complete while reporting
+ * the progress of indexes vacuumed.
+ *
+ */
+ if (VacuumActiveNWorkers)
+ {
+ while (pg_atomic_read_u32(VacuumActiveNWorkers) > 0)
+ {
+ parallel_vacuum_progress_report();
+ }
+ }
+
/*
* Next, accumulate buffer and WAL usage. (This must wait for the workers
* to finish, or we might get incomplete data.)
*/
if (nworkers > 0)
{
- /* Wait for all vacuum workers to finish */
+ /*
+ * Wait for all vacuum workers to finish
+ *
+ * We must do this even if we know we don't
+ * have anymore active workers. See the
+ * WaitForParallelWorkersToFinish commentary
+ * as to why.
+ */
WaitForParallelWorkersToFinish(pvs->pcxt);
for (int i = 0; i < pvs->pcxt->nworkers_launched; i++)
@@ -719,6 +771,14 @@ parallel_vacuum_process_all_indexes(ParallelVacuumState *pvs, int num_index_scan
VacuumSharedCostBalance = NULL;
VacuumActiveNWorkers = NULL;
}
+
+ /*
+ * Disabled index vacuum progress reporting.
+ * If we havee another index vacuum cycle, the
+ * progress reporting will be re-enabled then.
+ */
+ index_vacuum_completed = NULL;
+ report_parallel_vacuum_progress = false;
}
/*
@@ -823,6 +883,21 @@ parallel_vacuum_process_one_index(ParallelVacuumState *pvs, Relation indrel,
IndexBulkDeleteResult *istat = NULL;
IndexBulkDeleteResult *istat_res;
IndexVacuumInfo ivinfo;
+ Oid indrelid = RelationGetRelid(indrel);
+
+ /*
+ * If we are a parallel worker, start a PROGRESS_COMMAND_VACUUM_PARALLEL
+ * command for progress reporting, and set the leader pid.
+ */
+ if (IsParallelWorker())
+ {
+ pgstat_progress_start_command(PROGRESS_COMMAND_VACUUM_PARALLEL,
+ pvs->shared->relid);
+ pgstat_progress_update_param(PROGRESS_VACUUM_LEADER_PID, pvs->shared->leader_pid);
+ }
+
+ /* report the index being vacuumed or cleaned up */
+ pgstat_progress_update_param(PROGRESS_VACUUM_INDRELID, indrelid);
/*
* Update the pointer to the corresponding bulk-deletion result if someone
@@ -846,9 +921,11 @@ parallel_vacuum_process_one_index(ParallelVacuumState *pvs, Relation indrel,
switch (indstats->status)
{
case PARALLEL_INDVAC_STATUS_NEED_BULKDELETE:
+ pgstat_progress_update_param(PROGRESS_VACUUM_PHASE, PROGRESS_VACUUM_PHASE_VACUUM_INDEX);
istat_res = vac_bulkdel_one_index(&ivinfo, istat, pvs->dead_items);
break;
case PARALLEL_INDVAC_STATUS_NEED_CLEANUP:
+ pgstat_progress_update_param(PROGRESS_VACUUM_PHASE, PROGRESS_VACUUM_PHASE_INDEX_CLEANUP);
istat_res = vac_cleanup_one_index(&ivinfo, istat);
break;
default:
@@ -888,6 +965,17 @@ parallel_vacuum_process_one_index(ParallelVacuumState *pvs, Relation indrel,
pvs->status = PARALLEL_INDVAC_STATUS_COMPLETED;
pfree(pvs->indname);
pvs->indname = NULL;
+
+ /*
+ * Reset the index relid begin vacuumed and incremebts the
+ * index vacuum counter.
+ */
+ pgstat_progress_update_param(PROGRESS_VACUUM_INDRELID, 0);
+ pg_atomic_add_fetch_u32(&(pvs->shared->idx_completed_progress), 1);
+
+ /* if we are a parallel worker, end the command */
+ if (IsParallelWorker())
+ pgstat_progress_end_command();
}
/*
@@ -1072,3 +1160,17 @@ parallel_vacuum_error_callback(void *arg)
return;
}
}
+
+/*
+ * Read the number of indexes vacuumed from the shared counter
+ * and report it to progress.h
+ */
+void
+parallel_vacuum_progress_report(void)
+{
+ if (IsParallelWorker() || !report_parallel_vacuum_progress)
+ return;
+
+ pgstat_progress_update_param(PROGRESS_VACUUM_INDEX_COMPLETED,
+ pg_atomic_read_u32(index_vacuum_completed));
+}
diff --git a/src/backend/utils/adt/pgstatfuncs.c b/src/backend/utils/adt/pgstatfuncs.c
index eadd8464ff..1584a9bd7d 100644
--- a/src/backend/utils/adt/pgstatfuncs.c
+++ b/src/backend/utils/adt/pgstatfuncs.c
@@ -484,6 +484,8 @@ pg_stat_get_progress_info(PG_FUNCTION_ARGS)
cmdtype = PROGRESS_COMMAND_BASEBACKUP;
else if (pg_strcasecmp(cmd, "COPY") == 0)
cmdtype = PROGRESS_COMMAND_COPY;
+ else if (pg_strcasecmp(cmd, "VACUUM_PARALLEL") == 0)
+ cmdtype = PROGRESS_COMMAND_VACUUM_PARALLEL;
else
ereport(ERROR,
(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index a28938caf4..410eec217b 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -25,6 +25,10 @@
#define PROGRESS_VACUUM_NUM_INDEX_VACUUMS 4
#define PROGRESS_VACUUM_MAX_DEAD_TUPLES 5
#define PROGRESS_VACUUM_NUM_DEAD_TUPLES 6
+#define PROGRESS_VACUUM_INDRELID 7
+#define PROGRESS_VACUUM_LEADER_PID 8
+#define PROGRESS_VACUUM_INDEX_TOTAL 9
+#define PROGRESS_VACUUM_INDEX_COMPLETED 10
/* Phases of vacuum (as advertised via PROGRESS_VACUUM_PHASE) */
#define PROGRESS_VACUUM_PHASE_SCAN_HEAP 1
diff --git a/src/include/commands/vacuum.h b/src/include/commands/vacuum.h
index 5d816ba7f4..96901d7234 100644
--- a/src/include/commands/vacuum.h
+++ b/src/include/commands/vacuum.h
@@ -338,4 +338,6 @@ extern double anl_random_fract(void);
extern double anl_init_selection_state(int n);
extern double anl_get_next_S(double t, int n, double *stateptr);
+extern void parallel_vacuum_progress_report(void);
+
#endif /* VACUUM_H */
diff --git a/src/include/utils/backend_progress.h b/src/include/utils/backend_progress.h
index 47bf8029b0..74d9dfb4c7 100644
--- a/src/include/utils/backend_progress.h
+++ b/src/include/utils/backend_progress.h
@@ -17,6 +17,10 @@
/* ----------
* Command type for progress reporting purposes
+ *
+ * Note: PROGRESS_COMMAND_VACUUM_PARALLEL is not
+ * a command per se, but this type is used to track
+ * parallel vacuum workers progress.
* ----------
*/
typedef enum ProgressCommandType
@@ -27,7 +31,8 @@ typedef enum ProgressCommandType
PROGRESS_COMMAND_CLUSTER,
PROGRESS_COMMAND_CREATE_INDEX,
PROGRESS_COMMAND_BASEBACKUP,
- PROGRESS_COMMAND_COPY
+ PROGRESS_COMMAND_COPY,
+ PROGRESS_COMMAND_VACUUM_PARALLEL
} ProgressCommandType;
#define PGSTAT_NUM_PROGRESS_PARAM 20
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index 9dd137415e..d7d6148d52 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -2013,9 +2013,24 @@ pg_stat_progress_vacuum| SELECT s.pid,
s.param4 AS heap_blks_vacuumed,
s.param5 AS index_vacuum_count,
s.param6 AS max_dead_tuples,
- s.param7 AS num_dead_tuples
+ s.param7 AS num_dead_tuples,
+ s.param10 AS indexes_total,
+ s.param11 AS indexes_completed
FROM (pg_stat_get_progress_info('VACUUM'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
LEFT JOIN pg_database d ON ((s.datid = d.oid)));
+pg_stat_progress_vacuum_index| SELECT s.pid,
+ s.param9 AS leader_pid,
+ s.param8 AS indexrelid
+ FROM (pg_stat_get_progress_info('VACUUM'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
+ LEFT JOIN pg_database d ON ((s.datid = d.oid)))
+ WHERE ((s.param1 = ANY (ARRAY[(2)::bigint, (4)::bigint])) AND (s.param8 > 0))
+UNION ALL
+ SELECT s.pid,
+ s.param9 AS leader_pid,
+ s.param8 AS indexrelid
+ FROM (pg_stat_get_progress_info('VACUUM_PARALLEL'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
+ LEFT JOIN pg_database d ON ((s.datid = d.oid)))
+ WHERE ((s.param1 = ANY (ARRAY[(2)::bigint, (4)::bigint])) AND (s.param8 > 0));
pg_stat_recovery_prefetch| SELECT s.stats_reset,
s.prefetch,
s.hit,
--
2.32.1 (Apple Git-133)
^ permalink raw reply [nested|flat] 44+ messages in thread
* Re: Add index scan progress to pg_stat_progress_vacuum
@ 2022-10-11 13:50 Imseih (AWS), Sami <[email protected]>
parent: Masahiko Sawada <[email protected]>
1 sibling, 1 reply; 44+ messages in thread
From: Imseih (AWS), Sami @ 2022-10-11 13:50 UTC (permalink / raw)
To: Masahiko Sawada <[email protected]>; Robert Haas <[email protected]>; +Cc: Bossart, Nathan <[email protected]>; Andres Freund <[email protected]>; Peter Geoghegan <[email protected]>; Justin Pryzby <[email protected]>; pgsql-hackers
> One idea would be to add a flag, say report_parallel_vacuum_progress,
> to IndexVacuumInfo struct and expect index AM to check and update the
> parallel index vacuum progress, say every 1GB blocks processed. The
> flag is true only when the leader process is vacuuming an index.
> Regards,
Sorry for the long delay on this. I have taken the approach as suggested
by Sawada-san and Robert and attached is v12.
1. The patch introduces a new counter in the same shared memory already
used by the parallel leader and workers to keep track of the number
of indexes completed. This way there is no reason to loop through
the index status every time we want to get the status of indexes completed.
2. A new function in vacuumparallel.c will be used to update
the progress of indexes completed by reading from the
counter created in point #1.
3. The function is called during the vacuum_delay_point as a
matter of convenience, since this is called in all major vacuum
loops. The function will only do something if the caller
sets a boolean to report progress. Doing so will also ensure
progress is being reported in case the parallel workers completed
before the leader.
4. Rather than adding any complexity to WaitForParallelWorkersToFinish
and introducing a new callback, vacuumparallel.c will wait until
the number of vacuum workers is 0 and then call
WaitForParallelWorkersToFinish as it does currently.
5. Went back to the idea of adding a new view called pg_stat_progress_vacuum_index
which is accomplished by adding a new type called VACUUM_PARALLEL in progress.h
Thanks,
Sami Imseih
Amazon Web Servies (AWS)
FYI: the above message was originally sent yesterday but
was created under a separate thread. Please ignore this
thread[1]
[1]: https://www.postgresql.org/message-id/flat/4CD97E17-B9E4-421E-9A53-4317C90EFF35%40amazon.com
Attachments:
[application/octet-stream] v12-0001--Show-progress-for-index-vacuums.patch (23.0K, ../../[email protected]/2-v12-0001--Show-progress-for-index-vacuums.patch)
download | inline diff:
From fd394f0bf01406f850206a6c4a81ff187a685a69 Mon Sep 17 00:00:00 2001
From: "Imseih (AWS)" <[email protected]>
Date: Mon, 10 Oct 2022 11:22:25 -0500
Subject: [PATCH v12 1/1] Add 2 new columns to pg_stat_progress_vacuum. The
columns are indexes_total as the total indexes to be vacuumed or cleaned and
indexes_processed as the number of indexes vacuumed or cleaned up so far.
Also, introduce a new view called pg_stat_progress_vacuum_index that
exposes the current index being vacuumed.
Author: Sami Imseih, based on suggestions by Nathan Bossart, Peter Geoghegan and Masahiko Sawada
Reviewed by: Nathan Bossart, Masahiko Sawada
Discussion: https://www.postgresql.org/message-id/flat/[email protected]
---
doc/src/sgml/monitoring.sgml | 92 ++++++++++++++++++++++
doc/src/sgml/ref/vacuum.sgml | 8 +-
src/backend/access/heap/vacuumlazy.c | 44 ++++++++++-
src/backend/catalog/system_views.sql | 20 ++++-
src/backend/commands/vacuum.c | 6 ++
src/backend/commands/vacuumparallel.c | 106 +++++++++++++++++++++++++-
src/backend/utils/adt/pgstatfuncs.c | 2 +
src/include/commands/progress.h | 4 +
src/include/commands/vacuum.h | 2 +
src/include/utils/backend_progress.h | 7 +-
src/test/regress/expected/rules.out | 17 ++++-
11 files changed, 301 insertions(+), 7 deletions(-)
diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml
index 342b20ebeb..473c76e6e8 100644
--- a/doc/src/sgml/monitoring.sgml
+++ b/doc/src/sgml/monitoring.sgml
@@ -392,6 +392,15 @@ postgres 27093 0.0 0.0 30096 2752 ? Ss 11:34 0:00 postgres: ser
</entry>
</row>
+ <row>
+ <entry><structname>pg_stat_progress_vacuum_index</structname><indexterm><primary>pg_stat_progress_vacuum_index</primary>
+ </indexterm>
+ </entry>
+ <entry>One row for each backend (including autovacuum worker processes) performing the <literal>vacuuming indexes</literal>
+ or <literal>cleaning up indexes</literal> phase of a <command>VACUUM</command>.
+ </entry>
+ </row>
+
<row>
<entry><structname>pg_stat_progress_cluster</structname><indexterm><primary>pg_stat_progress_cluster</primary></indexterm></entry>
<entry>One row for each backend running
@@ -6414,6 +6423,89 @@ FROM pg_stat_get_backend_idset() AS backendid;
Number of dead tuples collected since the last index vacuum cycle.
</para></entry>
</row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>indexes_total</structfield> <type>bigint</type>
+ </para>
+ <para>
+ Number of indexes that wil be vacuumed. This value will be
+ <literal>0</literal> if there are no indexes to vacuum or
+ vacuum failsafe is triggered. See <xref linkend="guc-vacuum-failsafe-age"/>
+ for more on vacuum failsafe.
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>indexes_completed</structfield> <type>bigint</type>
+ </para>
+ <para>
+ Number of indexes vacuumed in the current vacuum cycle.
+ </para></entry>
+ </row>
+ </tbody>
+ </tgroup>
+ </table>
+
+ <indexterm>
+ <primary>pg_stat_progress_vacuum_index</primary>
+ </indexterm>
+
+ <para>
+ Whenever <command>VACUUM</command> is running, the
+ <structname>pg_stat_progress_vacuum_index</structname> view will contain
+ one row for each backend (including autovacuum worker processes) performing
+ the <literal>vacuuming indexes</literal> or <literal>cleaning up indexes</literal>
+ phase of a <command>VACUUM</command>.
+ </para>
+
+ <table id="pg-stat-progress-vacuum-view_index" xreflabel="pg_stat_progress_vacuum_index">
+ <title><structname>pg_stat_progress_vacuum_index</structname> View</title>
+ <tgroup cols="1">
+ <thead>
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ Column Type
+ </para>
+ <para>
+ Description
+ </para></entry>
+ </row>
+ </thead>
+
+ <tbody>
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>pid</structfield> <type>integer</type>
+ </para>
+ <para>
+ Process ID of backend.
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>leader_pid</structfield> <type>integer</type>
+ </para>
+ <para>
+ Process ID of the leader backend in a parallel <command>VACUUM</command>. This value
+ will match the <structfield>pid</structfield> value whenever the leader
+ is processing an index or the <command>VACUUM</command> is not using parallel.
+ This field can be joined to <structfield>pid</structfield>
+ of <structfield>pg_stat_progress_vacuum</structfield> to get more details about
+ the <command>VACUUM</command>.
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>indexrelid</structfield> <type>integer</type>
+ </para>
+ <para>
+ OID of the index being processed in the current vacuum phase.
+ </para></entry>
+ </row>
</tbody>
</tgroup>
</table>
diff --git a/doc/src/sgml/ref/vacuum.sgml b/doc/src/sgml/ref/vacuum.sgml
index c582021d29..0c08d9ac6d 100644
--- a/doc/src/sgml/ref/vacuum.sgml
+++ b/doc/src/sgml/ref/vacuum.sgml
@@ -411,7 +411,13 @@ VACUUM [ FULL ] [ FREEZE ] [ VERBOSE ] [ ANALYZE ] [ <replaceable class="paramet
<para>
Each backend running <command>VACUUM</command> without the
<literal>FULL</literal> option will report its progress in the
- <structname>pg_stat_progress_vacuum</structname> view. Backends running
+ <structname>pg_stat_progress_vacuum</structname> view.
+ <structname>pg_stat_progress_vacuum</structname> view. Whenever a
+ <command>VACUUM</command> is in the <literal>vacuuming indexes</literal>
+ or <literal>cleaning up indexes</literal> phase,
+ the current index being processed is reported in
+ <structname>pg_stat_progress_vacuum_index</structname>.
+ <command>VACUUM FULL</command> will report their progress in the Backends running
<command>VACUUM FULL</command> will instead report their progress in the
<structname>pg_stat_progress_cluster</structname> view. See
<xref linkend="vacuum-progress-reporting"/> and
diff --git a/src/backend/access/heap/vacuumlazy.c b/src/backend/access/heap/vacuumlazy.c
index dfbe37472f..b057d18dd9 100644
--- a/src/backend/access/heap/vacuumlazy.c
+++ b/src/backend/access/heap/vacuumlazy.c
@@ -350,8 +350,10 @@ heap_vacuum_rel(Relation rel, VacuumParams *params,
}
}
+ /* start the vacuum progress command and report the leader pid. */
pgstat_progress_start_command(PROGRESS_COMMAND_VACUUM,
RelationGetRelid(rel));
+ pgstat_progress_update_param(PROGRESS_VACUUM_LEADER_PID, MyProcPid);
/*
* Get OldestXmin cutoff, which is used to determine which deleted tuples
@@ -420,6 +422,10 @@ heap_vacuum_rel(Relation rel, VacuumParams *params,
vacrel->rel = rel;
vac_open_indexes(vacrel->rel, RowExclusiveLock, &vacrel->nindexes,
&vacrel->indrels);
+
+ /* report number of indexes to vacuum */
+ pgstat_progress_update_param(PROGRESS_VACUUM_INDEX_TOTAL, vacrel->nindexes);
+
if (instrument && vacrel->nindexes > 0)
{
/* Copy index names used by instrumentation (not error reporting) */
@@ -2337,10 +2343,21 @@ lazy_vacuum_all_indexes(LVRelState *vacrel)
Relation indrel = vacrel->indrels[idx];
IndexBulkDeleteResult *istat = vacrel->indstats[idx];
+ /* report the index relid being vacuumed */
+ pgstat_progress_update_param(PROGRESS_VACUUM_INDRELID, RelationGetRelid(indrel));
+
vacrel->indstats[idx] =
lazy_vacuum_one_index(indrel, istat, vacrel->old_live_tuples,
vacrel);
+ /*
+ * Done vacuuming an index.
+ * Increment the indexes completed and reset the index relid to 0
+ */
+ pgstat_progress_update_param(PROGRESS_VACUUM_INDEX_COMPLETED,
+ idx + 1);
+ pgstat_progress_update_param(PROGRESS_VACUUM_INDRELID, 0);
+
if (lazy_check_wraparound_failsafe(vacrel))
{
/* Wraparound emergency -- end current index scan */
@@ -2384,6 +2401,13 @@ lazy_vacuum_all_indexes(LVRelState *vacrel)
pgstat_progress_update_param(PROGRESS_VACUUM_NUM_INDEX_VACUUMS,
vacrel->num_index_scans);
+ /*
+ * Reset the indexes completed at this point.
+ * If we end up in another index vacuum cycle, we will
+ * start counting from the start.
+ */
+ pgstat_progress_update_param(PROGRESS_VACUUM_INDEX_COMPLETED, 0);
+
return allindexes;
}
@@ -2633,10 +2657,17 @@ lazy_check_wraparound_failsafe(LVRelState *vacrel)
{
vacrel->failsafe_active = true;
- /* Disable index vacuuming, index cleanup, and heap rel truncation */
+ /*
+ * Disable index vacuuming, index cleanup, and heap rel truncation
+ *
+ * Also, report to progress.h that we are no longer tracking
+ * index vacuum/cleanup.
+ */
vacrel->do_index_vacuuming = false;
vacrel->do_index_cleanup = false;
vacrel->do_rel_truncate = false;
+ pgstat_progress_update_param(PROGRESS_VACUUM_INDEX_TOTAL, 0);
+ pgstat_progress_update_param(PROGRESS_VACUUM_INDEX_COMPLETED, 0);
ereport(WARNING,
(errmsg("bypassing nonessential maintenance of table \"%s.%s.%s\" as a failsafe after %d index scans",
@@ -2681,9 +2712,20 @@ lazy_cleanup_all_indexes(LVRelState *vacrel)
Relation indrel = vacrel->indrels[idx];
IndexBulkDeleteResult *istat = vacrel->indstats[idx];
+ /*+ report the index relid being cleaned */
+ pgstat_progress_update_param(PROGRESS_VACUUM_INDRELID, RelationGetRelid(indrel));
+
vacrel->indstats[idx] =
lazy_cleanup_one_index(indrel, istat, reltuples,
estimated_count, vacrel);
+
+ /*
+ * Done cleaning an index.
+ * Increment the indexes completed and reset the index relid to 0
+ */
+ pgstat_progress_update_param(PROGRESS_VACUUM_INDEX_COMPLETED,
+ idx + 1);
+ pgstat_progress_update_param(PROGRESS_VACUUM_INDRELID, 0);
}
}
else
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index 55f7ec79e0..5b5e7b4080 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1162,10 +1162,28 @@ CREATE VIEW pg_stat_progress_vacuum AS
END AS phase,
S.param2 AS heap_blks_total, S.param3 AS heap_blks_scanned,
S.param4 AS heap_blks_vacuumed, S.param5 AS index_vacuum_count,
- S.param6 AS max_dead_tuples, S.param7 AS num_dead_tuples
+ S.param6 AS max_dead_tuples, S.param7 AS num_dead_tuples,
+ S.param10 AS indexes_total, S.param11 AS indexes_completed
FROM pg_stat_get_progress_info('VACUUM') AS S
LEFT JOIN pg_database D ON S.datid = D.oid;
+CREATE VIEW pg_stat_progress_vacuum_index AS
+ SELECT
+ S.pid AS pid,
+ S.param9 AS leader_pid,
+ S.param8 AS indexrelid
+ FROM pg_stat_get_progress_info('VACUUM') AS S
+ LEFT JOIN pg_database D ON S.datid = D.oid
+ WHERE S.param1 in (2, 4) AND S.param8 > 0
+ UNION ALL
+ SELECT
+ S.pid AS pid,
+ S.param9 AS leader_pid,
+ S.param8 AS indexrelid
+ FROM pg_stat_get_progress_info('VACUUM_PARALLEL') AS S
+ LEFT JOIN pg_database D ON S.datid = D.oid
+ WHERE S.param1 in (2, 4) AND S.param8 > 0;
+
CREATE VIEW pg_stat_progress_cluster AS
SELECT
S.pid AS pid,
diff --git a/src/backend/commands/vacuum.c b/src/backend/commands/vacuum.c
index 7ccde07de9..19a2692704 100644
--- a/src/backend/commands/vacuum.c
+++ b/src/backend/commands/vacuum.c
@@ -2173,12 +2173,18 @@ vac_close_indexes(int nindexes, Relation *Irel, LOCKMODE lockmode)
*
* This should be called in each major loop of VACUUM processing,
* typically once per page processed.
+ *
+ * NOTE: For convenience, parallel_vacuum_progress_report, is called
+ * here so the leader can report the number of indexes vacuumed in
+ * while inside all the major VACUUM loops.
*/
void
vacuum_delay_point(void)
{
double msec = 0;
+ parallel_vacuum_progress_report();
+
/* Always check for interrupts */
CHECK_FOR_INTERRUPTS();
diff --git a/src/backend/commands/vacuumparallel.c b/src/backend/commands/vacuumparallel.c
index f26d796e52..65a71cbac9 100644
--- a/src/backend/commands/vacuumparallel.c
+++ b/src/backend/commands/vacuumparallel.c
@@ -30,6 +30,7 @@
#include "access/table.h"
#include "access/xact.h"
#include "catalog/index.h"
+#include "commands/progress.h"
#include "commands/vacuum.h"
#include "optimizer/paths.h"
#include "pgstat.h"
@@ -103,6 +104,20 @@ typedef struct PVShared
/* Counter for vacuuming and cleanup */
pg_atomic_uint32 idx;
+
+ /*
+ * Counter for vacuuming and cleanup progress reporting.
+ * This value is used to report index vacuum/cleanup progress
+ * in parallel_vacuum_progress_report. We keep this
+ * counter to avoid having to loop through
+ * ParallelVacuumState->indstats to determine the number
+ * of indexes completed.
+ */
+ pg_atomic_uint32 idx_completed_progress;
+
+ /* track the leader pid of a parallel vacuum */
+ int leader_pid;
+
} PVShared;
/* Status used during parallel index vacuum or cleanup */
@@ -214,6 +229,9 @@ static bool parallel_vacuum_index_is_parallel_safe(Relation indrel, int num_inde
bool vacuum);
static void parallel_vacuum_error_callback(void *arg);
+static pg_atomic_uint32 *index_vacuum_completed = NULL;
+static bool report_parallel_vacuum_progress = false;
+
/*
* Try to enter parallel mode and create a parallel context. Then initialize
* shared memory state.
@@ -364,6 +382,9 @@ parallel_vacuum_init(Relation rel, Relation *indrels, int nindexes,
pg_atomic_init_u32(&(shared->cost_balance), 0);
pg_atomic_init_u32(&(shared->active_nworkers), 0);
pg_atomic_init_u32(&(shared->idx), 0);
+ pg_atomic_init_u32(&(shared->idx_completed_progress), 0);
+
+ shared->leader_pid = MyProcPid;
shm_toc_insert(pcxt->toc, PARALLEL_VACUUM_KEY_SHARED, shared);
pvs->shared = shared;
@@ -618,8 +639,9 @@ parallel_vacuum_process_all_indexes(ParallelVacuumState *pvs, int num_index_scan
vacuum));
}
- /* Reset the parallel index processing counter */
+ /* Reset the parallel index processing counter ( index proress counter also ) */
pg_atomic_write_u32(&(pvs->shared->idx), 0);
+ pg_atomic_write_u32(&(pvs->shared->idx_completed_progress), 0);
/* Setup the shared cost-based vacuum delay and launch workers */
if (nworkers > 0)
@@ -657,6 +679,13 @@ parallel_vacuum_process_all_indexes(ParallelVacuumState *pvs, int num_index_scan
/* Enable shared cost balance for leader backend */
VacuumSharedCostBalance = &(pvs->shared->cost_balance);
VacuumActiveNWorkers = &(pvs->shared->active_nworkers);
+
+ /*
+ * If we are launching a parallel vacuum/cleanup,
+ * setup the tracking.
+ */
+ report_parallel_vacuum_progress = true;
+ index_vacuum_completed = &(pvs->shared->idx_completed_progress);
}
if (vacuum)
@@ -682,13 +711,36 @@ parallel_vacuum_process_all_indexes(ParallelVacuumState *pvs, int num_index_scan
*/
parallel_vacuum_process_safe_indexes(pvs);
+ /*
+ * In case the leader completes vacuuming all
+ * its indexes before the parallel workers do,
+ * it can spin here waiting for the number of
+ * active workers to complete while reporting
+ * the progress of indexes vacuumed.
+ *
+ */
+ if (VacuumActiveNWorkers)
+ {
+ while (pg_atomic_read_u32(VacuumActiveNWorkers) > 0)
+ {
+ parallel_vacuum_progress_report();
+ }
+ }
+
/*
* Next, accumulate buffer and WAL usage. (This must wait for the workers
* to finish, or we might get incomplete data.)
*/
if (nworkers > 0)
{
- /* Wait for all vacuum workers to finish */
+ /*
+ * Wait for all vacuum workers to finish
+ *
+ * We must do this even if we know we don't
+ * have anymore active workers. See the
+ * WaitForParallelWorkersToFinish commentary
+ * as to why.
+ */
WaitForParallelWorkersToFinish(pvs->pcxt);
for (int i = 0; i < pvs->pcxt->nworkers_launched; i++)
@@ -719,6 +771,14 @@ parallel_vacuum_process_all_indexes(ParallelVacuumState *pvs, int num_index_scan
VacuumSharedCostBalance = NULL;
VacuumActiveNWorkers = NULL;
}
+
+ /*
+ * Disabled index vacuum progress reporting.
+ * If we havee another index vacuum cycle, the
+ * progress reporting will be re-enabled then.
+ */
+ index_vacuum_completed = NULL;
+ report_parallel_vacuum_progress = false;
}
/*
@@ -823,6 +883,21 @@ parallel_vacuum_process_one_index(ParallelVacuumState *pvs, Relation indrel,
IndexBulkDeleteResult *istat = NULL;
IndexBulkDeleteResult *istat_res;
IndexVacuumInfo ivinfo;
+ Oid indrelid = RelationGetRelid(indrel);
+
+ /*
+ * If we are a parallel worker, start a PROGRESS_COMMAND_VACUUM_PARALLEL
+ * command for progress reporting, and set the leader pid.
+ */
+ if (IsParallelWorker())
+ {
+ pgstat_progress_start_command(PROGRESS_COMMAND_VACUUM_PARALLEL,
+ pvs->shared->relid);
+ pgstat_progress_update_param(PROGRESS_VACUUM_LEADER_PID, pvs->shared->leader_pid);
+ }
+
+ /* report the index being vacuumed or cleaned up */
+ pgstat_progress_update_param(PROGRESS_VACUUM_INDRELID, indrelid);
/*
* Update the pointer to the corresponding bulk-deletion result if someone
@@ -846,9 +921,11 @@ parallel_vacuum_process_one_index(ParallelVacuumState *pvs, Relation indrel,
switch (indstats->status)
{
case PARALLEL_INDVAC_STATUS_NEED_BULKDELETE:
+ pgstat_progress_update_param(PROGRESS_VACUUM_PHASE, PROGRESS_VACUUM_PHASE_VACUUM_INDEX);
istat_res = vac_bulkdel_one_index(&ivinfo, istat, pvs->dead_items);
break;
case PARALLEL_INDVAC_STATUS_NEED_CLEANUP:
+ pgstat_progress_update_param(PROGRESS_VACUUM_PHASE, PROGRESS_VACUUM_PHASE_INDEX_CLEANUP);
istat_res = vac_cleanup_one_index(&ivinfo, istat);
break;
default:
@@ -888,6 +965,17 @@ parallel_vacuum_process_one_index(ParallelVacuumState *pvs, Relation indrel,
pvs->status = PARALLEL_INDVAC_STATUS_COMPLETED;
pfree(pvs->indname);
pvs->indname = NULL;
+
+ /*
+ * Reset the index relid begin vacuumed and incremebts the
+ * index vacuum counter.
+ */
+ pgstat_progress_update_param(PROGRESS_VACUUM_INDRELID, 0);
+ pg_atomic_add_fetch_u32(&(pvs->shared->idx_completed_progress), 1);
+
+ /* if we are a parallel worker, end the command */
+ if (IsParallelWorker())
+ pgstat_progress_end_command();
}
/*
@@ -1072,3 +1160,17 @@ parallel_vacuum_error_callback(void *arg)
return;
}
}
+
+/*
+ * Read the number of indexes vacuumed from the shared counter
+ * and report it to progress.h
+ */
+void
+parallel_vacuum_progress_report(void)
+{
+ if (IsParallelWorker() || !report_parallel_vacuum_progress)
+ return;
+
+ pgstat_progress_update_param(PROGRESS_VACUUM_INDEX_COMPLETED,
+ pg_atomic_read_u32(index_vacuum_completed));
+}
diff --git a/src/backend/utils/adt/pgstatfuncs.c b/src/backend/utils/adt/pgstatfuncs.c
index eadd8464ff..1584a9bd7d 100644
--- a/src/backend/utils/adt/pgstatfuncs.c
+++ b/src/backend/utils/adt/pgstatfuncs.c
@@ -484,6 +484,8 @@ pg_stat_get_progress_info(PG_FUNCTION_ARGS)
cmdtype = PROGRESS_COMMAND_BASEBACKUP;
else if (pg_strcasecmp(cmd, "COPY") == 0)
cmdtype = PROGRESS_COMMAND_COPY;
+ else if (pg_strcasecmp(cmd, "VACUUM_PARALLEL") == 0)
+ cmdtype = PROGRESS_COMMAND_VACUUM_PARALLEL;
else
ereport(ERROR,
(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index a28938caf4..410eec217b 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -25,6 +25,10 @@
#define PROGRESS_VACUUM_NUM_INDEX_VACUUMS 4
#define PROGRESS_VACUUM_MAX_DEAD_TUPLES 5
#define PROGRESS_VACUUM_NUM_DEAD_TUPLES 6
+#define PROGRESS_VACUUM_INDRELID 7
+#define PROGRESS_VACUUM_LEADER_PID 8
+#define PROGRESS_VACUUM_INDEX_TOTAL 9
+#define PROGRESS_VACUUM_INDEX_COMPLETED 10
/* Phases of vacuum (as advertised via PROGRESS_VACUUM_PHASE) */
#define PROGRESS_VACUUM_PHASE_SCAN_HEAP 1
diff --git a/src/include/commands/vacuum.h b/src/include/commands/vacuum.h
index 5d816ba7f4..96901d7234 100644
--- a/src/include/commands/vacuum.h
+++ b/src/include/commands/vacuum.h
@@ -338,4 +338,6 @@ extern double anl_random_fract(void);
extern double anl_init_selection_state(int n);
extern double anl_get_next_S(double t, int n, double *stateptr);
+extern void parallel_vacuum_progress_report(void);
+
#endif /* VACUUM_H */
diff --git a/src/include/utils/backend_progress.h b/src/include/utils/backend_progress.h
index 47bf8029b0..74d9dfb4c7 100644
--- a/src/include/utils/backend_progress.h
+++ b/src/include/utils/backend_progress.h
@@ -17,6 +17,10 @@
/* ----------
* Command type for progress reporting purposes
+ *
+ * Note: PROGRESS_COMMAND_VACUUM_PARALLEL is not
+ * a command per se, but this type is used to track
+ * parallel vacuum workers progress.
* ----------
*/
typedef enum ProgressCommandType
@@ -27,7 +31,8 @@ typedef enum ProgressCommandType
PROGRESS_COMMAND_CLUSTER,
PROGRESS_COMMAND_CREATE_INDEX,
PROGRESS_COMMAND_BASEBACKUP,
- PROGRESS_COMMAND_COPY
+ PROGRESS_COMMAND_COPY,
+ PROGRESS_COMMAND_VACUUM_PARALLEL
} ProgressCommandType;
#define PGSTAT_NUM_PROGRESS_PARAM 20
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index 9dd137415e..d7d6148d52 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -2013,9 +2013,24 @@ pg_stat_progress_vacuum| SELECT s.pid,
s.param4 AS heap_blks_vacuumed,
s.param5 AS index_vacuum_count,
s.param6 AS max_dead_tuples,
- s.param7 AS num_dead_tuples
+ s.param7 AS num_dead_tuples,
+ s.param10 AS indexes_total,
+ s.param11 AS indexes_completed
FROM (pg_stat_get_progress_info('VACUUM'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
LEFT JOIN pg_database d ON ((s.datid = d.oid)));
+pg_stat_progress_vacuum_index| SELECT s.pid,
+ s.param9 AS leader_pid,
+ s.param8 AS indexrelid
+ FROM (pg_stat_get_progress_info('VACUUM'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
+ LEFT JOIN pg_database d ON ((s.datid = d.oid)))
+ WHERE ((s.param1 = ANY (ARRAY[(2)::bigint, (4)::bigint])) AND (s.param8 > 0))
+UNION ALL
+ SELECT s.pid,
+ s.param9 AS leader_pid,
+ s.param8 AS indexrelid
+ FROM (pg_stat_get_progress_info('VACUUM_PARALLEL'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
+ LEFT JOIN pg_database d ON ((s.datid = d.oid)))
+ WHERE ((s.param1 = ANY (ARRAY[(2)::bigint, (4)::bigint])) AND (s.param8 > 0));
pg_stat_recovery_prefetch| SELECT s.stats_reset,
s.prefetch,
s.hit,
--
2.32.1 (Apple Git-133)
^ permalink raw reply [nested|flat] 44+ messages in thread
* Re: Add index scan progress to pg_stat_progress_vacuum
@ 2022-10-12 07:15 Masahiko Sawada <[email protected]>
parent: Imseih (AWS), Sami <[email protected]>
0 siblings, 1 reply; 44+ messages in thread
From: Masahiko Sawada @ 2022-10-12 07:15 UTC (permalink / raw)
To: Imseih (AWS), Sami <[email protected]>; +Cc: Robert Haas <[email protected]>; Bossart, Nathan <[email protected]>; Andres Freund <[email protected]>; Peter Geoghegan <[email protected]>; Justin Pryzby <[email protected]>; pgsql-hackers
On Tue, Oct 11, 2022 at 10:50 PM Imseih (AWS), Sami <[email protected]> wrote:
>
> > One idea would be to add a flag, say report_parallel_vacuum_progress,
> > to IndexVacuumInfo struct and expect index AM to check and update the
> > parallel index vacuum progress, say every 1GB blocks processed. The
> > flag is true only when the leader process is vacuuming an index.
>
> > Regards,
>
> Sorry for the long delay on this. I have taken the approach as suggested
> by Sawada-san and Robert and attached is v12.
Thank you for updating the patch!
>
> 1. The patch introduces a new counter in the same shared memory already
> used by the parallel leader and workers to keep track of the number
> of indexes completed. This way there is no reason to loop through
> the index status every time we want to get the status of indexes completed.
While it seems to be a good idea to have the atomic counter for the
number of indexes completed, I think we should not use the global
variable referencing the counter as follow:
+static pg_atomic_uint32 *index_vacuum_completed = NULL;
:
+void
+parallel_vacuum_progress_report(void)
+{
+ if (IsParallelWorker() || !report_parallel_vacuum_progress)
+ return;
+
+ pgstat_progress_update_param(PROGRESS_VACUUM_INDEX_COMPLETED,
+ pg_atomic_read_u32(index_vacuum_completed));
+}
I think we can pass ParallelVacuumState (or PVIndStats) to the
reporting function so that it can check the counter and report the
progress.
> 2. A new function in vacuumparallel.c will be used to update
> the progress of indexes completed by reading from the
> counter created in point #1.
>
> 3. The function is called during the vacuum_delay_point as a
> matter of convenience, since this is called in all major vacuum
> loops. The function will only do something if the caller
> sets a boolean to report progress. Doing so will also ensure
> progress is being reported in case the parallel workers completed
> before the leader.
Robert pointed out:
---
I am not too sure that the idea of using the vacuum delay points is the best
plan. I think we should try to avoid piggybacking on such general
infrastructure if we can, and instead look for a way to tie this to
something that is specific to parallel vacuum.
---
I agree with this part.
Instead, I think we can add a boolean and the pointer for
ParallelVacuumState to IndexVacuumInfo. If the boolean is true, an
index AM can call the reporting function with the pointer to
ParallelVacuumState while scanning index blocks, for example, for
every 1GB. The boolean can be true only for the leader process.
>
> 4. Rather than adding any complexity to WaitForParallelWorkersToFinish
> and introducing a new callback, vacuumparallel.c will wait until
> the number of vacuum workers is 0 and then call
> WaitForParallelWorkersToFinish as it does currently.
Agreed, but with the following change, the leader process waits in a
busy loop, which should not be acceptable:
+ if (VacuumActiveNWorkers)
+ {
+ while (pg_atomic_read_u32(VacuumActiveNWorkers) > 0)
+ {
+ parallel_vacuum_progress_report();
+ }
+ }
+
Also, I think it's better to check whether idx_completed_progress
equals to the number indexes instead.
> 5. Went back to the idea of adding a new view called pg_stat_progress_vacuum_index
> which is accomplished by adding a new type called VACUUM_PARALLEL in progress.h
Probably, we can devide the patch into two patches. One for adding the
new statistics of the number of vacuumed indexes to
pg_stat_progress_vacuum and another one for adding new statistics view
pg_stat_progress_vacuum_index.
Regards,
--
Masahiko Sawada
PostgreSQL Contributors Team
RDS Open Source Databases
Amazon Web Services: https://aws.amazon.com
^ permalink raw reply [nested|flat] 44+ messages in thread
* Re: Add index scan progress to pg_stat_progress_vacuum
@ 2022-10-14 20:05 Imseih (AWS), Sami <[email protected]>
parent: Masahiko Sawada <[email protected]>
0 siblings, 1 reply; 44+ messages in thread
From: Imseih (AWS), Sami @ 2022-10-14 20:05 UTC (permalink / raw)
To: Masahiko Sawada <[email protected]>; +Cc: Robert Haas <[email protected]>; Bossart, Nathan <[email protected]>; Andres Freund <[email protected]>; Peter Geoghegan <[email protected]>; Justin Pryzby <[email protected]>; pgsql-hackers
Thank you for the feedback!
> While it seems to be a good idea to have the atomic counter for the
> number of indexes completed, I think we should not use the global
> variable referencing the counter as follow:
> +static pg_atomic_uint32 *index_vacuum_completed = NULL;
> :
> +void
> +parallel_vacuum_progress_report(void)
> +{
> + if (IsParallelWorker() || !report_parallel_vacuum_progress)
> + return;
> +
> + pgstat_progress_update_param(PROGRESS_VACUUM_INDEX_COMPLETED,
> + pg_atomic_read_u32(index_vacuum_completed));
> +}
> I think we can pass ParallelVacuumState (or PVIndStats) to the
> reporting function so that it can check the counter and report the
> progress.
Yes, that makes sense.
> ---
> I am not too sure that the idea of using the vacuum delay points is the best
> plan. I think we should try to avoid piggybacking on such general
> infrastructure if we can, and instead look for a way to tie this to
> something that is specific to parallel vacuum.
> ---
Adding the call to vacuum_delay_point made sense since it's
called in all major vacuum scans. While it is also called
by analyze, it will only do anything if the caller sets a flag
to report_parallel_vacuum_progress.
> Instead, I think we can add a boolean and the pointer for
> ParallelVacuumState to IndexVacuumInfo. If the boolean is true, an
> index AM can call the reporting function with the pointer to
> ParallelVacuumState while scanning index blocks, for example, for
> every 1GB. The boolean can be true only for the leader process.
Will doing this every 1GB be necessary? Considering the function
will not do much more than update progress from the value
stored in index_vacuum_completed?
> Agreed, but with the following change, the leader process waits in a
> busy loop, which should not be acceptable:
Good point.
> Also, I think it's better to check whether idx_completed_progress
equals to the number indexes instead.
Good point
> 5. Went back to the idea of adding a new view called pg_stat_progress_vacuum_index
> which is accomplished by adding a new type called VACUUM_PARALLEL in progress.h
> Probably, we can devide the patch into two patches. One for adding the
Makes sense.
Thanks
Sami Imseih
Amazon Web Services (AWS)
^ permalink raw reply [nested|flat] 44+ messages in thread
* Re: Add index scan progress to pg_stat_progress_vacuum
@ 2022-11-02 16:52 Imseih (AWS), Sami <[email protected]>
parent: Imseih (AWS), Sami <[email protected]>
0 siblings, 3 replies; 44+ messages in thread
From: Imseih (AWS), Sami @ 2022-11-02 16:52 UTC (permalink / raw)
To: Masahiko Sawada <[email protected]>; +Cc: Robert Haas <[email protected]>; Bossart, Nathan <[email protected]>; Andres Freund <[email protected]>; Peter Geoghegan <[email protected]>; Justin Pryzby <[email protected]>; pgsql-hackers
Attached is v13-0001--Show-progress-for-index-vacuums.patch which addresses
the latest comments. The main changes are:
1/ Call the parallel_vacuum_progress_report inside the AMs rather than vacuum_delay_point.
2/ A Boolean when set to True in vacuumparallel.c will be used to determine if calling
parallel_vacuum_progress_report is necessary.
3/ Removed global varilable from vacuumparallel.c
4/ Went back to calling parallel_vacuum_progress_report inside
WaitForParallelWorkersToFinish to cover the case when a
leader is waiting for parallel workers to finish.
5/ I did not see a need to only report progress after 1GB as it's a fairly cheap call to update
progress.
6/ v1-0001-Function-to-return-currently-vacuumed-or-cleaned-ind.patch is a separate patch
for exposing the index relid being vacuumed by a backend.
Thanks
Sami Imseih
Amazon Web Services (AWS)
Attachments:
[application/octet-stream] v13-0001--Show-progress-for-index-vacuums.patch (24.3K, ../../[email protected]/2-v13-0001--Show-progress-for-index-vacuums.patch)
download | inline diff:
From e080c28d09567cb3dd974c0855487c717dad0a57 Mon Sep 17 00:00:00 2001
From: "Imseih (AWS)" <[email protected]>
Date: Tue, 1 Nov 2022 11:50:14 -0500
Subject: [PATCH 1/1] Add 2 new columns to pg_stat_progress_vacuum. The columns
are indexes_total as the total indexes to be vacuumed or cleaned and
indexes_processed as the number of indexes vacuumed or cleaned up so far.
Author: Sami Imseih, based on suggestions by Nathan Bossart, Peter Geoghegan and Masahiko Sawada
Reviewed by: Nathan Bossart, Masahiko Sawada
Discussion: https://www.postgresql.org/message-id/flat/[email protected]
---
contrib/bloom/blvacuum.c | 6 +++
doc/src/sgml/monitoring.sgml | 21 +++++++++
src/backend/access/brin/brin.c | 9 ++--
src/backend/access/gin/ginvacuum.c | 9 ++++
src/backend/access/gist/gistvacuum.c | 5 +++
src/backend/access/hash/hash.c | 8 +++-
src/backend/access/hash/hashpage.c | 4 +-
src/backend/access/heap/vacuumlazy.c | 40 ++++++++++++++++-
src/backend/access/nbtree/nbtree.c | 3 ++
src/backend/access/spgist/spgvacuum.c | 3 ++
src/backend/access/transam/parallel.c | 5 +++
src/backend/catalog/index.c | 1 +
src/backend/catalog/system_views.sql | 3 +-
src/backend/commands/vacuumparallel.c | 63 ++++++++++++++++++++++++++-
src/include/access/genam.h | 3 ++
src/include/access/hash.h | 3 +-
src/include/access/parallel.h | 2 +
src/include/commands/progress.h | 2 +
src/include/commands/vacuum.h | 2 +
src/test/regress/expected/rules.out | 4 +-
20 files changed, 184 insertions(+), 12 deletions(-)
diff --git a/contrib/bloom/blvacuum.c b/contrib/bloom/blvacuum.c
index 91fae5b0c0..19ae819cfe 100644
--- a/contrib/bloom/blvacuum.c
+++ b/contrib/bloom/blvacuum.c
@@ -136,6 +136,9 @@ blbulkdelete(IndexVacuumInfo *info, IndexBulkDeleteResult *stats,
GenericXLogAbort(gxlogState);
}
UnlockReleaseBuffer(buffer);
+
+ if (info->report_parallel_progress)
+ info->parallel_progress_callback(info->parallel_progress_arg);
}
/*
@@ -209,6 +212,9 @@ blvacuumcleanup(IndexVacuumInfo *info, IndexBulkDeleteResult *stats)
}
UnlockReleaseBuffer(buffer);
+
+ if (info->report_parallel_progress)
+ info->parallel_progress_callback(info->parallel_progress_arg);
}
IndexFreeSpaceMapVacuum(info->index);
diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml
index e5d622d514..c61443f3d5 100644
--- a/doc/src/sgml/monitoring.sgml
+++ b/doc/src/sgml/monitoring.sgml
@@ -6455,6 +6455,27 @@ FROM pg_stat_get_backend_idset() AS backendid;
Number of dead tuples collected since the last index vacuum cycle.
</para></entry>
</row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>indexes_total</structfield> <type>bigint</type>
+ </para>
+ <para>
+ Number of indexes that wil be vacuumed. This value will be
+ <literal>0</literal> if there are no indexes to vacuum or
+ vacuum failsafe is triggered. See <xref linkend="guc-vacuum-failsafe-age"/>
+ for more on vacuum failsafe.
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>indexes_completed</structfield> <type>bigint</type>
+ </para>
+ <para>
+ Number of indexes vacuumed in the current vacuum cycle.
+ </para></entry>
+ </row>
</tbody>
</tgroup>
</table>
diff --git a/src/backend/access/brin/brin.c b/src/backend/access/brin/brin.c
index 20b7d65b94..bd96ecf41e 100644
--- a/src/backend/access/brin/brin.c
+++ b/src/backend/access/brin/brin.c
@@ -78,7 +78,7 @@ static void brinsummarize(Relation index, Relation heapRel, BlockNumber pageRang
static void form_and_insert_tuple(BrinBuildState *state);
static void union_tuples(BrinDesc *bdesc, BrinMemTuple *a,
BrinTuple *b);
-static void brin_vacuum_scan(Relation idxrel, BufferAccessStrategy strategy);
+static void brin_vacuum_scan(Relation idxrel, BufferAccessStrategy strategy, IndexVacuumInfo *info);
static bool add_values_to_range(Relation idxRel, BrinDesc *bdesc,
BrinMemTuple *dtup, Datum *values, bool *nulls);
static bool check_null_keys(BrinValues *bval, ScanKey *nullkeys, int nnullkeys);
@@ -952,7 +952,7 @@ brinvacuumcleanup(IndexVacuumInfo *info, IndexBulkDeleteResult *stats)
heapRel = table_open(IndexGetRelation(RelationGetRelid(info->index), false),
AccessShareLock);
- brin_vacuum_scan(info->index, info->strategy);
+ brin_vacuum_scan(info->index, info->strategy, info);
brinsummarize(info->index, heapRel, BRIN_ALL_BLOCKRANGES, false,
&stats->num_index_tuples, &stats->num_index_tuples);
@@ -1659,7 +1659,7 @@ union_tuples(BrinDesc *bdesc, BrinMemTuple *a, BrinTuple *b)
* and such.
*/
static void
-brin_vacuum_scan(Relation idxrel, BufferAccessStrategy strategy)
+brin_vacuum_scan(Relation idxrel, BufferAccessStrategy strategy, IndexVacuumInfo *info)
{
BlockNumber nblocks;
BlockNumber blkno;
@@ -1681,6 +1681,9 @@ brin_vacuum_scan(Relation idxrel, BufferAccessStrategy strategy)
brin_page_cleanup(idxrel, buf);
ReleaseBuffer(buf);
+
+ if (info->report_parallel_progress)
+ info->parallel_progress_callback(info->parallel_progress_arg);
}
/*
diff --git a/src/backend/access/gin/ginvacuum.c b/src/backend/access/gin/ginvacuum.c
index b4fa5f6bf8..3d5e4600dc 100644
--- a/src/backend/access/gin/ginvacuum.c
+++ b/src/backend/access/gin/ginvacuum.c
@@ -633,6 +633,9 @@ ginbulkdelete(IndexVacuumInfo *info, IndexBulkDeleteResult *stats,
UnlockReleaseBuffer(buffer);
buffer = ReadBufferExtended(index, MAIN_FORKNUM, blkno,
RBM_NORMAL, info->strategy);
+
+ if (info->report_parallel_progress)
+ info->parallel_progress_callback(info->parallel_progress_arg);
}
/* right now we found leftmost page in entry's BTree */
@@ -677,6 +680,9 @@ ginbulkdelete(IndexVacuumInfo *info, IndexBulkDeleteResult *stats,
buffer = ReadBufferExtended(index, MAIN_FORKNUM, blkno,
RBM_NORMAL, info->strategy);
LockBuffer(buffer, GIN_EXCLUSIVE);
+
+ if (info->report_parallel_progress)
+ info->parallel_progress_callback(info->parallel_progress_arg);
}
MemoryContextDelete(gvs.tmpCxt);
@@ -775,6 +781,9 @@ ginvacuumcleanup(IndexVacuumInfo *info, IndexBulkDeleteResult *stats)
}
UnlockReleaseBuffer(buffer);
+
+ if (info->report_parallel_progress)
+ info->parallel_progress_callback(info->parallel_progress_arg);
}
/* Update the metapage with accurate page and entry counts */
diff --git a/src/backend/access/gist/gistvacuum.c b/src/backend/access/gist/gistvacuum.c
index 0aa6e58a62..da3a0540d0 100644
--- a/src/backend/access/gist/gistvacuum.c
+++ b/src/backend/access/gist/gistvacuum.c
@@ -223,7 +223,12 @@ gistvacuumscan(IndexVacuumInfo *info, IndexBulkDeleteResult *stats,
break;
/* Iterate over pages, then loop back to recheck length */
for (; blkno < num_pages; blkno++)
+ {
gistvacuumpage(&vstate, blkno, blkno);
+
+ if (info->report_parallel_progress)
+ info->parallel_progress_callback(info->parallel_progress_arg);
+ }
}
/*
diff --git a/src/backend/access/hash/hash.c b/src/backend/access/hash/hash.c
index c361509d68..14790adf61 100644
--- a/src/backend/access/hash/hash.c
+++ b/src/backend/access/hash/hash.c
@@ -550,7 +550,7 @@ loop_top:
cachedmetap->hashm_highmask,
cachedmetap->hashm_lowmask, &tuples_removed,
&num_index_tuples, split_cleanup,
- callback, callback_state);
+ callback, callback_state, info);
_hash_dropbuf(rel, bucket_buf);
@@ -686,7 +686,8 @@ hashbucketcleanup(Relation rel, Bucket cur_bucket, Buffer bucket_buf,
uint32 maxbucket, uint32 highmask, uint32 lowmask,
double *tuples_removed, double *num_index_tuples,
bool split_cleanup,
- IndexBulkDeleteCallback callback, void *callback_state)
+ IndexBulkDeleteCallback callback, void *callback_state,
+ IndexVacuumInfo *info)
{
BlockNumber blkno;
Buffer buf;
@@ -775,6 +776,9 @@ hashbucketcleanup(Relation rel, Bucket cur_bucket, Buffer bucket_buf,
if (num_index_tuples)
*num_index_tuples += 1;
}
+
+ if (info && info->report_parallel_progress)
+ info->parallel_progress_callback(info->parallel_progress_arg);
}
/* retain the pin on primary bucket page till end of bucket scan */
diff --git a/src/backend/access/hash/hashpage.c b/src/backend/access/hash/hashpage.c
index d2edcd4617..e9ce1a1110 100644
--- a/src/backend/access/hash/hashpage.c
+++ b/src/backend/access/hash/hashpage.c
@@ -759,7 +759,7 @@ restart_expand:
hashbucketcleanup(rel, old_bucket, buf_oblkno, start_oblkno, NULL,
maxbucket, highmask, lowmask, NULL, NULL, true,
- NULL, NULL);
+ NULL, NULL, NULL);
_hash_dropbuf(rel, buf_oblkno);
@@ -1327,7 +1327,7 @@ _hash_splitbucket(Relation rel,
hashbucketcleanup(rel, obucket, bucket_obuf,
BufferGetBlockNumber(bucket_obuf), NULL,
maxbucket, highmask, lowmask, NULL, NULL, true,
- NULL, NULL);
+ NULL, NULL, NULL);
}
else
{
diff --git a/src/backend/access/heap/vacuumlazy.c b/src/backend/access/heap/vacuumlazy.c
index dfbe37472f..120a2b18d7 100644
--- a/src/backend/access/heap/vacuumlazy.c
+++ b/src/backend/access/heap/vacuumlazy.c
@@ -420,6 +420,10 @@ heap_vacuum_rel(Relation rel, VacuumParams *params,
vacrel->rel = rel;
vac_open_indexes(vacrel->rel, RowExclusiveLock, &vacrel->nindexes,
&vacrel->indrels);
+
+ /* report number of indexes to vacuum */
+ pgstat_progress_update_param(PROGRESS_VACUUM_INDEX_TOTAL, vacrel->nindexes);
+
if (instrument && vacrel->nindexes > 0)
{
/* Copy index names used by instrumentation (not error reporting) */
@@ -2341,6 +2345,12 @@ lazy_vacuum_all_indexes(LVRelState *vacrel)
lazy_vacuum_one_index(indrel, istat, vacrel->old_live_tuples,
vacrel);
+ /*
+ * Done vacuuming an index. Increment the indexes completed
+ */
+ pgstat_progress_update_param(PROGRESS_VACUUM_INDEX_COMPLETED,
+ idx + 1);
+
if (lazy_check_wraparound_failsafe(vacrel))
{
/* Wraparound emergency -- end current index scan */
@@ -2384,6 +2394,13 @@ lazy_vacuum_all_indexes(LVRelState *vacrel)
pgstat_progress_update_param(PROGRESS_VACUUM_NUM_INDEX_VACUUMS,
vacrel->num_index_scans);
+ /*
+ * Reset the indexes completed at this point.
+ * If we end up in another index vacuum cycle, we will
+ * start counting from the start.
+ */
+ pgstat_progress_update_param(PROGRESS_VACUUM_INDEX_COMPLETED, 0);
+
return allindexes;
}
@@ -2633,10 +2650,17 @@ lazy_check_wraparound_failsafe(LVRelState *vacrel)
{
vacrel->failsafe_active = true;
- /* Disable index vacuuming, index cleanup, and heap rel truncation */
+ /*
+ * Disable index vacuuming, index cleanup, and heap rel truncation
+ *
+ * Also, report to progress.h that we are no longer tracking
+ * index vacuum/cleanup.
+ */
vacrel->do_index_vacuuming = false;
vacrel->do_index_cleanup = false;
vacrel->do_rel_truncate = false;
+ pgstat_progress_update_param(PROGRESS_VACUUM_INDEX_TOTAL, 0);
+ pgstat_progress_update_param(PROGRESS_VACUUM_INDEX_COMPLETED, 0);
ereport(WARNING,
(errmsg("bypassing nonessential maintenance of table \"%s.%s.%s\" as a failsafe after %d index scans",
@@ -2684,6 +2708,12 @@ lazy_cleanup_all_indexes(LVRelState *vacrel)
vacrel->indstats[idx] =
lazy_cleanup_one_index(indrel, istat, reltuples,
estimated_count, vacrel);
+
+ /*
+ * Done cleaning an index. Increment the indexes completed
+ */
+ pgstat_progress_update_param(PROGRESS_VACUUM_INDEX_COMPLETED,
+ idx + 1);
}
}
else
@@ -2693,6 +2723,12 @@ lazy_cleanup_all_indexes(LVRelState *vacrel)
vacrel->num_index_scans,
estimated_count);
}
+
+ /*
+ * Reset the indexes completed at this point.
+ */
+ pgstat_progress_update_param(PROGRESS_VACUUM_INDEX_COMPLETED, 0);
+
}
/*
@@ -2718,6 +2754,7 @@ lazy_vacuum_one_index(Relation indrel, IndexBulkDeleteResult *istat,
ivinfo.index = indrel;
ivinfo.analyze_only = false;
ivinfo.report_progress = false;
+ ivinfo.report_parallel_progress = false;
ivinfo.estimated_count = true;
ivinfo.message_level = DEBUG2;
ivinfo.num_heap_tuples = reltuples;
@@ -2766,6 +2803,7 @@ lazy_cleanup_one_index(Relation indrel, IndexBulkDeleteResult *istat,
ivinfo.index = indrel;
ivinfo.analyze_only = false;
ivinfo.report_progress = false;
+ ivinfo.report_parallel_progress = false;
ivinfo.estimated_count = estimated_count;
ivinfo.message_level = DEBUG2;
diff --git a/src/backend/access/nbtree/nbtree.c b/src/backend/access/nbtree/nbtree.c
index b52eca8f38..9904a9eb0c 100644
--- a/src/backend/access/nbtree/nbtree.c
+++ b/src/backend/access/nbtree/nbtree.c
@@ -998,6 +998,9 @@ btvacuumscan(IndexVacuumInfo *info, IndexBulkDeleteResult *stats,
if (info->report_progress)
pgstat_progress_update_param(PROGRESS_SCAN_BLOCKS_DONE,
scanblkno);
+
+ if (info->report_parallel_progress)
+ info->parallel_progress_callback(info->parallel_progress_arg);
}
}
diff --git a/src/backend/access/spgist/spgvacuum.c b/src/backend/access/spgist/spgvacuum.c
index 0049630532..b1ab491980 100644
--- a/src/backend/access/spgist/spgvacuum.c
+++ b/src/backend/access/spgist/spgvacuum.c
@@ -843,6 +843,9 @@ spgvacuumscan(spgBulkDeleteState *bds)
/* empty the pending-list after each page */
if (bds->pendingList != NULL)
spgprocesspending(bds);
+
+ if (bds->info->report_parallel_progress)
+ bds->info->parallel_progress_callback(bds->info->parallel_progress_arg);
}
}
diff --git a/src/backend/access/transam/parallel.c b/src/backend/access/transam/parallel.c
index ee0985c7ed..0b4bde8297 100644
--- a/src/backend/access/transam/parallel.c
+++ b/src/backend/access/transam/parallel.c
@@ -185,6 +185,8 @@ CreateParallelContext(const char *library_name, const char *function_name,
pcxt->library_name = pstrdup(library_name);
pcxt->function_name = pstrdup(function_name);
pcxt->error_context_stack = error_context_stack;
+ pcxt->parallel_progress_callback = NULL;
+ pcxt->parallel_progress_arg = NULL;
shm_toc_initialize_estimator(&pcxt->estimator);
dlist_push_head(&pcxt_list, &pcxt->node);
@@ -785,6 +787,9 @@ WaitForParallelWorkersToFinish(ParallelContext *pcxt)
*/
CHECK_FOR_INTERRUPTS();
+ if (pcxt->parallel_progress_callback)
+ pcxt->parallel_progress_callback(pcxt->parallel_progress_arg);
+
for (i = 0; i < pcxt->nworkers_launched; ++i)
{
/*
diff --git a/src/backend/catalog/index.c b/src/backend/catalog/index.c
index 61f1d3926a..11b32129a7 100644
--- a/src/backend/catalog/index.c
+++ b/src/backend/catalog/index.c
@@ -3348,6 +3348,7 @@ validate_index(Oid heapId, Oid indexId, Snapshot snapshot)
ivinfo.index = indexRelation;
ivinfo.analyze_only = false;
ivinfo.report_progress = true;
+ ivinfo.report_parallel_progress = false;
ivinfo.estimated_count = true;
ivinfo.message_level = DEBUG2;
ivinfo.num_heap_tuples = heapRelation->rd_rel->reltuples;
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index 2d8104b090..c37b20b91b 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1165,7 +1165,8 @@ CREATE VIEW pg_stat_progress_vacuum AS
END AS phase,
S.param2 AS heap_blks_total, S.param3 AS heap_blks_scanned,
S.param4 AS heap_blks_vacuumed, S.param5 AS index_vacuum_count,
- S.param6 AS max_dead_tuples, S.param7 AS num_dead_tuples
+ S.param6 AS max_dead_tuples, S.param7 AS num_dead_tuples,
+ S.param8 AS indexes_total, S.param9 AS indexes_completed
FROM pg_stat_get_progress_info('VACUUM') AS S
LEFT JOIN pg_database D ON S.datid = D.oid;
diff --git a/src/backend/commands/vacuumparallel.c b/src/backend/commands/vacuumparallel.c
index f26d796e52..85e2de5d54 100644
--- a/src/backend/commands/vacuumparallel.c
+++ b/src/backend/commands/vacuumparallel.c
@@ -30,6 +30,7 @@
#include "access/table.h"
#include "access/xact.h"
#include "catalog/index.h"
+#include "commands/progress.h"
#include "commands/vacuum.h"
#include "optimizer/paths.h"
#include "pgstat.h"
@@ -103,6 +104,17 @@ typedef struct PVShared
/* Counter for vacuuming and cleanup */
pg_atomic_uint32 idx;
+
+ /*
+ * Counter for vacuuming and cleanup progress reporting.
+ * This value is used to report index vacuum/cleanup progress
+ * in parallel_vacuum_progress_report. We keep this
+ * counter to avoid having to loop through
+ * ParallelVacuumState->indstats to determine the number
+ * of indexes completed.
+ */
+ pg_atomic_uint32 idx_completed_progress;
+
} PVShared;
/* Status used during parallel index vacuum or cleanup */
@@ -273,6 +285,9 @@ parallel_vacuum_init(Relation rel, Relation *indrels, int nindexes,
Assert(pcxt->nworkers > 0);
pvs->pcxt = pcxt;
+ pvs->pcxt->parallel_progress_callback = parallel_vacuum_progress_report;
+ pvs->pcxt->parallel_progress_arg = pvs;
+
/* Estimate size for index vacuum stats -- PARALLEL_VACUUM_KEY_INDEX_STATS */
est_indstats_len = mul_size(sizeof(PVIndStats), nindexes);
shm_toc_estimate_chunk(&pcxt->estimator, est_indstats_len);
@@ -364,6 +379,7 @@ parallel_vacuum_init(Relation rel, Relation *indrels, int nindexes,
pg_atomic_init_u32(&(shared->cost_balance), 0);
pg_atomic_init_u32(&(shared->active_nworkers), 0);
pg_atomic_init_u32(&(shared->idx), 0);
+ pg_atomic_init_u32(&(shared->idx_completed_progress), 0);
shm_toc_insert(pcxt->toc, PARALLEL_VACUUM_KEY_SHARED, shared);
pvs->shared = shared;
@@ -618,8 +634,9 @@ parallel_vacuum_process_all_indexes(ParallelVacuumState *pvs, int num_index_scan
vacuum));
}
- /* Reset the parallel index processing counter */
+ /* Reset the parallel index processing counter ( index progress counter also ) */
pg_atomic_write_u32(&(pvs->shared->idx), 0);
+ pg_atomic_write_u32(&(pvs->shared->idx_completed_progress), 0);
/* Setup the shared cost-based vacuum delay and launch workers */
if (nworkers > 0)
@@ -834,10 +851,13 @@ parallel_vacuum_process_one_index(ParallelVacuumState *pvs, Relation indrel,
ivinfo.index = indrel;
ivinfo.analyze_only = false;
ivinfo.report_progress = false;
+ ivinfo.report_parallel_progress = true;
ivinfo.message_level = DEBUG2;
ivinfo.estimated_count = pvs->shared->estimated_count;
ivinfo.num_heap_tuples = pvs->shared->reltuples;
ivinfo.strategy = pvs->bstrategy;
+ ivinfo.parallel_progress_callback = parallel_vacuum_progress_report;
+ ivinfo.parallel_progress_arg = pvs;
/* Update error traceback information */
pvs->indname = pstrdup(RelationGetRelationName(indrel));
@@ -846,9 +866,11 @@ parallel_vacuum_process_one_index(ParallelVacuumState *pvs, Relation indrel,
switch (indstats->status)
{
case PARALLEL_INDVAC_STATUS_NEED_BULKDELETE:
+ pgstat_progress_update_param(PROGRESS_VACUUM_PHASE, PROGRESS_VACUUM_PHASE_VACUUM_INDEX);
istat_res = vac_bulkdel_one_index(&ivinfo, istat, pvs->dead_items);
break;
case PARALLEL_INDVAC_STATUS_NEED_CLEANUP:
+ pgstat_progress_update_param(PROGRESS_VACUUM_PHASE, PROGRESS_VACUUM_PHASE_INDEX_CLEANUP);
istat_res = vac_cleanup_one_index(&ivinfo, istat);
break;
default:
@@ -888,6 +910,11 @@ parallel_vacuum_process_one_index(ParallelVacuumState *pvs, Relation indrel,
pvs->status = PARALLEL_INDVAC_STATUS_COMPLETED;
pfree(pvs->indname);
pvs->indname = NULL;
+
+ /*
+ * Update the number of indexes completed.
+ */
+ pg_atomic_add_fetch_u32(&(pvs->shared->idx_completed_progress), 1);
}
/*
@@ -1072,3 +1099,37 @@ parallel_vacuum_error_callback(void *arg)
return;
}
}
+
+/*
+ * Read the number of indexes vacuumed from the shared counter
+ * and report it to progress.h
+ *
+ * This function is only called by the leader process of
+ * a parallel vacuum.
+ *
+ * If report_parallel_progress is set to true,
+ * this function is called from major loops inside the
+ * ambulkdelete and amvacuumcleanup.
+ *
+ * This function is also called inside WaitForParallelWorkersToFinish
+ * when report_parallel_progress is set to true.
+ *
+ * The reason for calling this function in both vacuum AM's
+ * and WaitForParallelWorkersToFinish is to ensure that
+ * parallel vacuum progress is constantly being reported
+ * if the leader process is either waiting for parallel
+ * proceses to finish (WaitForParallelWorkersToFinish) or
+ * the leader process is the last to finish and still
+ * inside the vacuum AM's.
+ */
+void
+parallel_vacuum_progress_report(void *arg)
+{
+ ParallelVacuumState *pvs = (ParallelVacuumState *) arg;
+
+ if (IsParallelWorker())
+ return;
+
+ pgstat_progress_update_param(PROGRESS_VACUUM_INDEX_COMPLETED,
+ pg_atomic_read_u32(&(pvs->shared->idx_completed_progress)));
+}
diff --git a/src/include/access/genam.h b/src/include/access/genam.h
index e1c4fdbd03..6d5fc189f4 100644
--- a/src/include/access/genam.h
+++ b/src/include/access/genam.h
@@ -46,10 +46,13 @@ typedef struct IndexVacuumInfo
Relation index; /* the index being vacuumed */
bool analyze_only; /* ANALYZE (without any actual vacuum) */
bool report_progress; /* emit progress.h status reports */
+ bool report_parallel_progress; /* emit progress.h status reports for parallel vacuum */
bool estimated_count; /* num_heap_tuples is an estimate */
int message_level; /* ereport level for progress messages */
double num_heap_tuples; /* tuples remaining in heap */
BufferAccessStrategy strategy; /* access strategy for reads */
+ void (*parallel_progress_callback)(void *arg);
+ void *parallel_progress_arg;
} IndexVacuumInfo;
/*
diff --git a/src/include/access/hash.h b/src/include/access/hash.h
index da372841c4..14fb75a4ad 100644
--- a/src/include/access/hash.h
+++ b/src/include/access/hash.h
@@ -480,6 +480,7 @@ extern void hashbucketcleanup(Relation rel, Bucket cur_bucket,
uint32 maxbucket, uint32 highmask, uint32 lowmask,
double *tuples_removed, double *num_index_tuples,
bool split_cleanup,
- IndexBulkDeleteCallback callback, void *callback_state);
+ IndexBulkDeleteCallback callback, void *callback_state,
+ IndexVacuumInfo *info);
#endif /* HASH_H */
diff --git a/src/include/access/parallel.h b/src/include/access/parallel.h
index 1ec8e33af4..77e9c0a517 100644
--- a/src/include/access/parallel.h
+++ b/src/include/access/parallel.h
@@ -46,6 +46,8 @@ typedef struct ParallelContext
ParallelWorkerInfo *worker;
int nknown_attached_workers;
bool *known_attached_workers;
+ void (*parallel_progress_callback)(void *arg);
+ void *parallel_progress_arg;
} ParallelContext;
typedef struct ParallelWorkerContext
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index a28938caf4..0e97c6d4ef 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -25,6 +25,8 @@
#define PROGRESS_VACUUM_NUM_INDEX_VACUUMS 4
#define PROGRESS_VACUUM_MAX_DEAD_TUPLES 5
#define PROGRESS_VACUUM_NUM_DEAD_TUPLES 6
+#define PROGRESS_VACUUM_INDEX_TOTAL 7
+#define PROGRESS_VACUUM_INDEX_COMPLETED 8
/* Phases of vacuum (as advertised via PROGRESS_VACUUM_PHASE) */
#define PROGRESS_VACUUM_PHASE_SCAN_HEAP 1
diff --git a/src/include/commands/vacuum.h b/src/include/commands/vacuum.h
index 5d816ba7f4..9510d028d3 100644
--- a/src/include/commands/vacuum.h
+++ b/src/include/commands/vacuum.h
@@ -338,4 +338,6 @@ extern double anl_random_fract(void);
extern double anl_init_selection_state(int n);
extern double anl_get_next_S(double t, int n, double *stateptr);
+extern void parallel_vacuum_progress_report(void *arg);
+
#endif /* VACUUM_H */
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index 624d0e5aae..e0bf81247f 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -2018,7 +2018,9 @@ pg_stat_progress_vacuum| SELECT s.pid,
s.param4 AS heap_blks_vacuumed,
s.param5 AS index_vacuum_count,
s.param6 AS max_dead_tuples,
- s.param7 AS num_dead_tuples
+ s.param7 AS num_dead_tuples,
+ s.param8 AS indexes_total,
+ s.param9 AS indexes_completed
FROM (pg_stat_get_progress_info('VACUUM'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
LEFT JOIN pg_database d ON ((s.datid = d.oid)));
pg_stat_recovery_prefetch| SELECT s.stats_reset,
--
2.32.1 (Apple Git-133)
[application/octet-stream] v1-0001-Function-to-return-currently-vacuumed-or-cleaned-ind.patch (10.2K, ../../[email protected]/3-v1-0001-Function-to-return-currently-vacuumed-or-cleaned-ind.patch)
download | inline diff:
From 03f716bb9b1a79884655d87a6b3986a622aa4285 Mon Sep 17 00:00:00 2001
From: "Imseih (AWS)" <[email protected]>
Date: Wed, 2 Nov 2022 10:36:10 -0500
Subject: [PATCH 1/1] Function to return currently vacuumed ( or cleaned )
index
pg_stat_get_vacuum_index takes a PID as an argument and
returns the index relid of the currently vacuumed index.
NULL is returned if the backend is not involved in an index vacuum.
---
doc/src/sgml/func.sgml | 13 +++++++
doc/src/sgml/monitoring.sgml | 8 +++--
src/backend/access/heap/vacuumlazy.c | 9 +++++
src/backend/commands/vacuumparallel.c | 23 ++++++++++++
src/backend/utils/adt/pgstatfuncs.c | 52 +++++++++++++++++++++++++++
src/include/catalog/pg_proc.dat | 4 +++
src/include/commands/progress.h | 1 +
src/include/utils/backend_progress.h | 7 +++-
8 files changed, 114 insertions(+), 3 deletions(-)
diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml
index 6e0425cb3d..39841354de 100644
--- a/doc/src/sgml/func.sgml
+++ b/doc/src/sgml/func.sgml
@@ -22670,6 +22670,19 @@ SELECT * FROM pg_ls_dir('.') WITH ORDINALITY AS t(ls,n);
parsing the text version.
</para></entry>
</row>
+
+ <row>
+ <entry role="func_table_entry"><para role="func_signature">
+ <indexterm>
+ <primary>pg_stat_get_vacuum_index</primary>
+ </indexterm>
+ <function>pg_stat_get_vacuum_index</function> ( <type>integer</type> )
+ <returnvalue>oid</returnvalue>
+ </para>
+ <para>
+ Returns the relid of an index current being vacuumed or cleaned by the backend.
+ </para></entry>
+ </row>
</tbody>
</tgroup>
</table>
diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml
index c61443f3d5..8d947d8c3a 100644
--- a/doc/src/sgml/monitoring.sgml
+++ b/doc/src/sgml/monitoring.sgml
@@ -6517,7 +6517,9 @@ FROM pg_stat_get_backend_idset() AS backendid;
has been completely scanned. It may happen multiple times per vacuum
if <xref linkend="guc-maintenance-work-mem"/> (or, in the case of autovacuum,
<xref linkend="guc-autovacuum-work-mem"/> if set) is insufficient to store
- the number of dead tuples found.
+ the number of dead tuples found. While in this phase, the function
+ <function>pg_stat_get_vacuum_index()</function> will return the relid
+ of the index being vacuumed.
</entry>
</row>
<row>
@@ -6536,7 +6538,9 @@ FROM pg_stat_get_backend_idset() AS backendid;
<entry>
<command>VACUUM</command> is currently cleaning up indexes. This occurs after
the heap has been completely scanned and all vacuuming of the indexes
- and the heap has been completed.
+ and the heap has been completed. While in this phase, the function
+ <function>pg_stat_get_vacuum_index()</function> will return the relid
+ of the index being cleaned.
</entry>
</row>
<row>
diff --git a/src/backend/access/heap/vacuumlazy.c b/src/backend/access/heap/vacuumlazy.c
index 120a2b18d7..8cd8e34f88 100644
--- a/src/backend/access/heap/vacuumlazy.c
+++ b/src/backend/access/heap/vacuumlazy.c
@@ -350,6 +350,7 @@ heap_vacuum_rel(Relation rel, VacuumParams *params,
}
}
+ /* start the vacuum progress command and report to progress.h */
pgstat_progress_start_command(PROGRESS_COMMAND_VACUUM,
RelationGetRelid(rel));
@@ -2341,6 +2342,9 @@ lazy_vacuum_all_indexes(LVRelState *vacrel)
Relation indrel = vacrel->indrels[idx];
IndexBulkDeleteResult *istat = vacrel->indstats[idx];
+ /* report the index relid being vacuumed */
+ pgstat_progress_update_param(PROGRESS_VACUUM_INDRELID, RelationGetRelid(indrel));
+
vacrel->indstats[idx] =
lazy_vacuum_one_index(indrel, istat, vacrel->old_live_tuples,
vacrel);
@@ -2350,6 +2354,7 @@ lazy_vacuum_all_indexes(LVRelState *vacrel)
*/
pgstat_progress_update_param(PROGRESS_VACUUM_INDEX_COMPLETED,
idx + 1);
+ pgstat_progress_update_param(PROGRESS_VACUUM_INDRELID, 0);
if (lazy_check_wraparound_failsafe(vacrel))
{
@@ -2705,6 +2710,9 @@ lazy_cleanup_all_indexes(LVRelState *vacrel)
Relation indrel = vacrel->indrels[idx];
IndexBulkDeleteResult *istat = vacrel->indstats[idx];
+ /* report the index relid being cleaned */
+ pgstat_progress_update_param(PROGRESS_VACUUM_INDRELID, RelationGetRelid(indrel));
+
vacrel->indstats[idx] =
lazy_cleanup_one_index(indrel, istat, reltuples,
estimated_count, vacrel);
@@ -2714,6 +2722,7 @@ lazy_cleanup_all_indexes(LVRelState *vacrel)
*/
pgstat_progress_update_param(PROGRESS_VACUUM_INDEX_COMPLETED,
idx + 1);
+ pgstat_progress_update_param(PROGRESS_VACUUM_INDRELID, 0);
}
}
else
diff --git a/src/backend/commands/vacuumparallel.c b/src/backend/commands/vacuumparallel.c
index 85e2de5d54..f2f51e86eb 100644
--- a/src/backend/commands/vacuumparallel.c
+++ b/src/backend/commands/vacuumparallel.c
@@ -840,6 +840,20 @@ parallel_vacuum_process_one_index(ParallelVacuumState *pvs, Relation indrel,
IndexBulkDeleteResult *istat = NULL;
IndexBulkDeleteResult *istat_res;
IndexVacuumInfo ivinfo;
+ Oid indrelid = RelationGetRelid(indrel);
+
+ /*
+ * If we are a parallel worker, start a PROGRESS_COMMAND_VACUUM_PARALLEL
+ * command for progress reporting and report the index being vacuumed.
+ */
+ if (IsParallelWorker())
+ {
+ pgstat_progress_start_command(PROGRESS_COMMAND_VACUUM_PARALLEL,
+ pvs->shared->relid);
+ }
+
+ /* report the index being vacuumed or cleaned up */
+ pgstat_progress_update_param(PROGRESS_VACUUM_INDRELID, indrelid);
/*
* Update the pointer to the corresponding bulk-deletion result if someone
@@ -911,10 +925,19 @@ parallel_vacuum_process_one_index(ParallelVacuumState *pvs, Relation indrel,
pfree(pvs->indname);
pvs->indname = NULL;
+ /*
+ * Reset the index relid begin vacuumed/cleaned
+ */
+ pgstat_progress_update_param(PROGRESS_VACUUM_INDRELID, 0);
+
/*
* Update the number of indexes completed.
*/
pg_atomic_add_fetch_u32(&(pvs->shared->idx_completed_progress), 1);
+
+ /* if we are a parallel worker, end the command */
+ if (IsParallelWorker())
+ pgstat_progress_end_command();
}
/*
diff --git a/src/backend/utils/adt/pgstatfuncs.c b/src/backend/utils/adt/pgstatfuncs.c
index 96bffc0f2a..b08c15e7aa 100644
--- a/src/backend/utils/adt/pgstatfuncs.c
+++ b/src/backend/utils/adt/pgstatfuncs.c
@@ -19,6 +19,7 @@
#include "access/xlogprefetcher.h"
#include "catalog/pg_authid.h"
#include "catalog/pg_type.h"
+#include "commands/progress.h"
#include "common/ip.h"
#include "funcapi.h"
#include "miscadmin.h"
@@ -2383,3 +2384,54 @@ pg_stat_have_stats(PG_FUNCTION_ARGS)
PG_RETURN_BOOL(pgstat_have_entry(kind, dboid, objoid));
}
+
+/*
+ * For a given PID, return the relid of the index
+ * currently being vacuumed.
+ */
+Datum
+pg_stat_get_vacuum_index(PG_FUNCTION_ARGS)
+{
+ int pid = PG_GETARG_INT32(0);
+ int num_backends = pgstat_fetch_stat_numbackends();
+ int curr_backend;
+ Oid indrelid = 0;
+ ProgressCommandType cmdtype_vacuum = PROGRESS_COMMAND_VACUUM;
+ ProgressCommandType cmdtype_vacuum_parallel = PROGRESS_COMMAND_VACUUM_PARALLEL;
+
+ /* 1-based index */
+ for (curr_backend = 1; curr_backend <= num_backends; curr_backend++)
+ {
+ LocalPgBackendStatus *local_beentry;
+ PgBackendStatus *beentry;
+
+ local_beentry = pgstat_fetch_stat_local_beentry(curr_backend);
+ beentry = &local_beentry->backendStatus;
+
+ /*
+ * If a backend has a VACUUM or VACUUM_PARALLEL progress command type
+ * and the procpid matches the given PID, return the index relid id
+ * that is current being vacuumed. In some cases, the index relid could
+ * be a '0' if we got here at thhe end of a vacuum or when no indexes
+ * are being vacuumed. In this case return NULL.
+ */
+ if (
+ (beentry->st_progress_command == cmdtype_vacuum
+ || beentry->st_progress_command == cmdtype_vacuum_parallel) &&
+ beentry->st_procpid == pid
+ )
+ {
+ indrelid = Int64GetDatum(beentry->st_progress_param[PROGRESS_VACUUM_INDRELID]);
+
+ if (indrelid > 0)
+ PG_RETURN_OID(indrelid);
+ else
+ PG_RETURN_NULL();
+ }
+ }
+
+ /*
+ * No matching PID performing a VACUUM operation, return NULL.
+ */
+ PG_RETURN_NULL();
+}
\ No newline at end of file
diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat
index 20f5aa56ea..dd5a32172a 100644
--- a/src/include/catalog/pg_proc.dat
+++ b/src/include/catalog/pg_proc.dat
@@ -11827,5 +11827,9 @@
proname => 'brin_minmax_multi_summary_send', provolatile => 's',
prorettype => 'bytea', proargtypes => 'pg_brin_minmax_multi_summary',
prosrc => 'brin_minmax_multi_summary_send' },
+{ oid => '4642',
+ descr => 'get the relid of the index being vacuumed by a backend',
+ proname => 'pg_stat_get_vacuum_index', provolatile => 'v', prorettype => 'int8',
+ proargtypes => 'int4', prosrc => 'pg_stat_get_vacuum_index' },
]
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 0e97c6d4ef..bc2b7d1e95 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -27,6 +27,7 @@
#define PROGRESS_VACUUM_NUM_DEAD_TUPLES 6
#define PROGRESS_VACUUM_INDEX_TOTAL 7
#define PROGRESS_VACUUM_INDEX_COMPLETED 8
+#define PROGRESS_VACUUM_INDRELID 9
/* Phases of vacuum (as advertised via PROGRESS_VACUUM_PHASE) */
#define PROGRESS_VACUUM_PHASE_SCAN_HEAP 1
diff --git a/src/include/utils/backend_progress.h b/src/include/utils/backend_progress.h
index 47bf8029b0..74d9dfb4c7 100644
--- a/src/include/utils/backend_progress.h
+++ b/src/include/utils/backend_progress.h
@@ -17,6 +17,10 @@
/* ----------
* Command type for progress reporting purposes
+ *
+ * Note: PROGRESS_COMMAND_VACUUM_PARALLEL is not
+ * a command per se, but this type is used to track
+ * parallel vacuum workers progress.
* ----------
*/
typedef enum ProgressCommandType
@@ -27,7 +31,8 @@ typedef enum ProgressCommandType
PROGRESS_COMMAND_CLUSTER,
PROGRESS_COMMAND_CREATE_INDEX,
PROGRESS_COMMAND_BASEBACKUP,
- PROGRESS_COMMAND_COPY
+ PROGRESS_COMMAND_COPY,
+ PROGRESS_COMMAND_VACUUM_PARALLEL
} ProgressCommandType;
#define PGSTAT_NUM_PROGRESS_PARAM 20
--
2.32.1 (Apple Git-133)
^ permalink raw reply [nested|flat] 44+ messages in thread
* Re: Add index scan progress to pg_stat_progress_vacuum
@ 2022-11-03 08:16 Ian Lawrence Barwick <[email protected]>
parent: Imseih (AWS), Sami <[email protected]>
2 siblings, 0 replies; 44+ messages in thread
From: Ian Lawrence Barwick @ 2022-11-03 08:16 UTC (permalink / raw)
To: Imseih (AWS), Sami <[email protected]>; +Cc: Masahiko Sawada <[email protected]>; Robert Haas <[email protected]>; Bossart, Nathan <[email protected]>; Andres Freund <[email protected]>; Peter Geoghegan <[email protected]>; Justin Pryzby <[email protected]>; pgsql-hackers
2022年11月3日(木) 1:52 Imseih (AWS), Sami <[email protected]>:
>
> Attached is v13-0001--Show-progress-for-index-vacuums.patch which addresses
> the latest comments. The main changes are:
>
> 1/ Call the parallel_vacuum_progress_report inside the AMs rather than vacuum_delay_point.
>
> 2/ A Boolean when set to True in vacuumparallel.c will be used to determine if calling
> parallel_vacuum_progress_report is necessary.
>
> 3/ Removed global varilable from vacuumparallel.c
>
> 4/ Went back to calling parallel_vacuum_progress_report inside
> WaitForParallelWorkersToFinish to cover the case when a
> leader is waiting for parallel workers to finish.
>
> 5/ I did not see a need to only report progress after 1GB as it's a fairly cheap call to update
> progress.
>
> 6/ v1-0001-Function-to-return-currently-vacuumed-or-cleaned-ind.patch is a separate patch
> for exposing the index relid being vacuumed by a backend.
This entry was marked "Needs review" in the CommitFest app but cfbot
reports the patch [1] no longer applies.
[1] this patch:
v1-0001-Function-to-return-currently-vacuumed-or-cleaned-ind.patch
We've marked it as "Waiting on Author". As CommitFest 2022-11 is
currently underway, this would be an excellent time update the patch.
Once you think the patchset is ready for review again, you (or any
interested party) can move the patch entry forward by visiting
https://commitfest.postgresql.org/40/3617/
and changing the status to "Needs review".
Thanks
Ian Barwick
^ permalink raw reply [nested|flat] 44+ messages in thread
* Re: Add index scan progress to pg_stat_progress_vacuum
@ 2022-11-04 13:27 Imseih (AWS), Sami <[email protected]>
parent: Imseih (AWS), Sami <[email protected]>
2 siblings, 1 reply; 44+ messages in thread
From: Imseih (AWS), Sami @ 2022-11-04 13:27 UTC (permalink / raw)
To: Masahiko Sawada <[email protected]>; +Cc: Robert Haas <[email protected]>; Bossart, Nathan <[email protected]>; Andres Freund <[email protected]>; Peter Geoghegan <[email protected]>; Justin Pryzby <[email protected]>; pgsql-hackers
Resubmitting patches with proper format.
Thanks
Sami Imseih
Amazon Web Services (AWS)
Attachments:
[application/octet-stream] v14-0002-Function-to-return-currently-vacuumed-or-cleaned.patch (10.3K, ../../[email protected]/2-v14-0002-Function-to-return-currently-vacuumed-or-cleaned.patch)
download | inline diff:
From 78a437946d5ccc70db98da9cc0fa0ae413afa625 Mon Sep 17 00:00:00 2001
From: "Imseih (AWS)" <[email protected]>
Date: Fri, 4 Nov 2022 07:52:02 -0500
Subject: [PATCH v14 2/2] Function to return currently vacuumed ( or cleaned )
index.
pg_stat_get_vacuum_index takes a PID as an argument and
returns the index relid of the currently vacuumed index.
NULL is returned if the backend is not involved in an index vacuum.
---
doc/src/sgml/func.sgml | 13 +++++++
doc/src/sgml/monitoring.sgml | 8 +++--
src/backend/access/heap/vacuumlazy.c | 9 +++++
src/backend/commands/vacuumparallel.c | 23 ++++++++++++
src/backend/utils/adt/pgstatfuncs.c | 52 +++++++++++++++++++++++++++
src/include/catalog/pg_proc.dat | 4 +++
src/include/commands/progress.h | 1 +
src/include/utils/backend_progress.h | 7 +++-
8 files changed, 114 insertions(+), 3 deletions(-)
22.2% doc/src/sgml/
11.2% src/backend/access/heap/
16.7% src/backend/commands/
37.1% src/backend/utils/adt/
5.7% src/include/catalog/
5.4% src/include/utils/
diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml
index 6e0425cb3d..39841354de 100644
--- a/doc/src/sgml/func.sgml
+++ b/doc/src/sgml/func.sgml
@@ -22670,6 +22670,19 @@ SELECT * FROM pg_ls_dir('.') WITH ORDINALITY AS t(ls,n);
parsing the text version.
</para></entry>
</row>
+
+ <row>
+ <entry role="func_table_entry"><para role="func_signature">
+ <indexterm>
+ <primary>pg_stat_get_vacuum_index</primary>
+ </indexterm>
+ <function>pg_stat_get_vacuum_index</function> ( <type>integer</type> )
+ <returnvalue>oid</returnvalue>
+ </para>
+ <para>
+ Returns the relid of an index current being vacuumed or cleaned by the backend.
+ </para></entry>
+ </row>
</tbody>
</tgroup>
</table>
diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml
index c61443f3d5..8d947d8c3a 100644
--- a/doc/src/sgml/monitoring.sgml
+++ b/doc/src/sgml/monitoring.sgml
@@ -6517,7 +6517,9 @@ FROM pg_stat_get_backend_idset() AS backendid;
has been completely scanned. It may happen multiple times per vacuum
if <xref linkend="guc-maintenance-work-mem"/> (or, in the case of autovacuum,
<xref linkend="guc-autovacuum-work-mem"/> if set) is insufficient to store
- the number of dead tuples found.
+ the number of dead tuples found. While in this phase, the function
+ <function>pg_stat_get_vacuum_index()</function> will return the relid
+ of the index being vacuumed.
</entry>
</row>
<row>
@@ -6536,7 +6538,9 @@ FROM pg_stat_get_backend_idset() AS backendid;
<entry>
<command>VACUUM</command> is currently cleaning up indexes. This occurs after
the heap has been completely scanned and all vacuuming of the indexes
- and the heap has been completed.
+ and the heap has been completed. While in this phase, the function
+ <function>pg_stat_get_vacuum_index()</function> will return the relid
+ of the index being cleaned.
</entry>
</row>
<row>
diff --git a/src/backend/access/heap/vacuumlazy.c b/src/backend/access/heap/vacuumlazy.c
index 120a2b18d7..8cd8e34f88 100644
--- a/src/backend/access/heap/vacuumlazy.c
+++ b/src/backend/access/heap/vacuumlazy.c
@@ -350,6 +350,7 @@ heap_vacuum_rel(Relation rel, VacuumParams *params,
}
}
+ /* start the vacuum progress command and report to progress.h */
pgstat_progress_start_command(PROGRESS_COMMAND_VACUUM,
RelationGetRelid(rel));
@@ -2341,6 +2342,9 @@ lazy_vacuum_all_indexes(LVRelState *vacrel)
Relation indrel = vacrel->indrels[idx];
IndexBulkDeleteResult *istat = vacrel->indstats[idx];
+ /* report the index relid being vacuumed */
+ pgstat_progress_update_param(PROGRESS_VACUUM_INDRELID, RelationGetRelid(indrel));
+
vacrel->indstats[idx] =
lazy_vacuum_one_index(indrel, istat, vacrel->old_live_tuples,
vacrel);
@@ -2350,6 +2354,7 @@ lazy_vacuum_all_indexes(LVRelState *vacrel)
*/
pgstat_progress_update_param(PROGRESS_VACUUM_INDEX_COMPLETED,
idx + 1);
+ pgstat_progress_update_param(PROGRESS_VACUUM_INDRELID, 0);
if (lazy_check_wraparound_failsafe(vacrel))
{
@@ -2705,6 +2710,9 @@ lazy_cleanup_all_indexes(LVRelState *vacrel)
Relation indrel = vacrel->indrels[idx];
IndexBulkDeleteResult *istat = vacrel->indstats[idx];
+ /* report the index relid being cleaned */
+ pgstat_progress_update_param(PROGRESS_VACUUM_INDRELID, RelationGetRelid(indrel));
+
vacrel->indstats[idx] =
lazy_cleanup_one_index(indrel, istat, reltuples,
estimated_count, vacrel);
@@ -2714,6 +2722,7 @@ lazy_cleanup_all_indexes(LVRelState *vacrel)
*/
pgstat_progress_update_param(PROGRESS_VACUUM_INDEX_COMPLETED,
idx + 1);
+ pgstat_progress_update_param(PROGRESS_VACUUM_INDRELID, 0);
}
}
else
diff --git a/src/backend/commands/vacuumparallel.c b/src/backend/commands/vacuumparallel.c
index 85e2de5d54..f2f51e86eb 100644
--- a/src/backend/commands/vacuumparallel.c
+++ b/src/backend/commands/vacuumparallel.c
@@ -840,6 +840,20 @@ parallel_vacuum_process_one_index(ParallelVacuumState *pvs, Relation indrel,
IndexBulkDeleteResult *istat = NULL;
IndexBulkDeleteResult *istat_res;
IndexVacuumInfo ivinfo;
+ Oid indrelid = RelationGetRelid(indrel);
+
+ /*
+ * If we are a parallel worker, start a PROGRESS_COMMAND_VACUUM_PARALLEL
+ * command for progress reporting and report the index being vacuumed.
+ */
+ if (IsParallelWorker())
+ {
+ pgstat_progress_start_command(PROGRESS_COMMAND_VACUUM_PARALLEL,
+ pvs->shared->relid);
+ }
+
+ /* report the index being vacuumed or cleaned up */
+ pgstat_progress_update_param(PROGRESS_VACUUM_INDRELID, indrelid);
/*
* Update the pointer to the corresponding bulk-deletion result if someone
@@ -911,10 +925,19 @@ parallel_vacuum_process_one_index(ParallelVacuumState *pvs, Relation indrel,
pfree(pvs->indname);
pvs->indname = NULL;
+ /*
+ * Reset the index relid begin vacuumed/cleaned
+ */
+ pgstat_progress_update_param(PROGRESS_VACUUM_INDRELID, 0);
+
/*
* Update the number of indexes completed.
*/
pg_atomic_add_fetch_u32(&(pvs->shared->idx_completed_progress), 1);
+
+ /* if we are a parallel worker, end the command */
+ if (IsParallelWorker())
+ pgstat_progress_end_command();
}
/*
diff --git a/src/backend/utils/adt/pgstatfuncs.c b/src/backend/utils/adt/pgstatfuncs.c
index 96bffc0f2a..82eedae1f0 100644
--- a/src/backend/utils/adt/pgstatfuncs.c
+++ b/src/backend/utils/adt/pgstatfuncs.c
@@ -19,6 +19,7 @@
#include "access/xlogprefetcher.h"
#include "catalog/pg_authid.h"
#include "catalog/pg_type.h"
+#include "commands/progress.h"
#include "common/ip.h"
#include "funcapi.h"
#include "miscadmin.h"
@@ -2383,3 +2384,54 @@ pg_stat_have_stats(PG_FUNCTION_ARGS)
PG_RETURN_BOOL(pgstat_have_entry(kind, dboid, objoid));
}
+
+/*
+ * For a given PID, return the relid of the index
+ * currently being vacuumed.
+ */
+Datum
+pg_stat_get_vacuum_index(PG_FUNCTION_ARGS)
+{
+ int pid = PG_GETARG_INT32(0);
+ int num_backends = pgstat_fetch_stat_numbackends();
+ int curr_backend;
+ Oid indrelid = 0;
+ ProgressCommandType cmdtype_vacuum = PROGRESS_COMMAND_VACUUM;
+ ProgressCommandType cmdtype_vacuum_parallel = PROGRESS_COMMAND_VACUUM_PARALLEL;
+
+ /* 1-based index */
+ for (curr_backend = 1; curr_backend <= num_backends; curr_backend++)
+ {
+ LocalPgBackendStatus *local_beentry;
+ PgBackendStatus *beentry;
+
+ local_beentry = pgstat_fetch_stat_local_beentry(curr_backend);
+ beentry = &local_beentry->backendStatus;
+
+ /*
+ * If a backend has a VACUUM or VACUUM_PARALLEL progress command type
+ * and the procpid matches the given PID, return the index relid
+ * that is current being vacuumed. In some cases, the index relid could
+ * be a '0' if we got here at the end of a vacuum or when no indexes
+ * are being vacuumed. In this case return NULL.
+ */
+ if (
+ (beentry->st_progress_command == cmdtype_vacuum
+ || beentry->st_progress_command == cmdtype_vacuum_parallel) &&
+ beentry->st_procpid == pid
+ )
+ {
+ indrelid = Int64GetDatum(beentry->st_progress_param[PROGRESS_VACUUM_INDRELID]);
+
+ if (indrelid > 0)
+ PG_RETURN_OID(indrelid);
+ else
+ PG_RETURN_NULL();
+ }
+ }
+
+ /*
+ * No matching PID performing a VACUUM operation, return NULL.
+ */
+ PG_RETURN_NULL();
+}
diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat
index 20f5aa56ea..dd5a32172a 100644
--- a/src/include/catalog/pg_proc.dat
+++ b/src/include/catalog/pg_proc.dat
@@ -11827,5 +11827,9 @@
proname => 'brin_minmax_multi_summary_send', provolatile => 's',
prorettype => 'bytea', proargtypes => 'pg_brin_minmax_multi_summary',
prosrc => 'brin_minmax_multi_summary_send' },
+{ oid => '4642',
+ descr => 'get the relid of the index being vacuumed by a backend',
+ proname => 'pg_stat_get_vacuum_index', provolatile => 'v', prorettype => 'int8',
+ proargtypes => 'int4', prosrc => 'pg_stat_get_vacuum_index' },
]
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 0e97c6d4ef..bc2b7d1e95 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -27,6 +27,7 @@
#define PROGRESS_VACUUM_NUM_DEAD_TUPLES 6
#define PROGRESS_VACUUM_INDEX_TOTAL 7
#define PROGRESS_VACUUM_INDEX_COMPLETED 8
+#define PROGRESS_VACUUM_INDRELID 9
/* Phases of vacuum (as advertised via PROGRESS_VACUUM_PHASE) */
#define PROGRESS_VACUUM_PHASE_SCAN_HEAP 1
diff --git a/src/include/utils/backend_progress.h b/src/include/utils/backend_progress.h
index 47bf8029b0..74d9dfb4c7 100644
--- a/src/include/utils/backend_progress.h
+++ b/src/include/utils/backend_progress.h
@@ -17,6 +17,10 @@
/* ----------
* Command type for progress reporting purposes
+ *
+ * Note: PROGRESS_COMMAND_VACUUM_PARALLEL is not
+ * a command per se, but this type is used to track
+ * parallel vacuum workers progress.
* ----------
*/
typedef enum ProgressCommandType
@@ -27,7 +31,8 @@ typedef enum ProgressCommandType
PROGRESS_COMMAND_CLUSTER,
PROGRESS_COMMAND_CREATE_INDEX,
PROGRESS_COMMAND_BASEBACKUP,
- PROGRESS_COMMAND_COPY
+ PROGRESS_COMMAND_COPY,
+ PROGRESS_COMMAND_VACUUM_PARALLEL
} ProgressCommandType;
#define PGSTAT_NUM_PROGRESS_PARAM 20
--
2.32.1 (Apple Git-133)
[application/octet-stream] v14-0001-Add-2-new-columns-to-pg_stat_progress_vacuum.-Th.patch (24.5K, ../../[email protected]/3-v14-0001-Add-2-new-columns-to-pg_stat_progress_vacuum.-Th.patch)
download | inline diff:
From d0dba3c28dbf8915db4df731edb445e5064f97bb Mon Sep 17 00:00:00 2001
From: "Imseih (AWS)" <[email protected]>
Date: Fri, 4 Nov 2022 07:48:42 -0500
Subject: [PATCH v14 1/2] Add 2 new columns to pg_stat_progress_vacuum. The
columns are indexes_total as the total indexes to be vacuumed or cleaned and
indexes_processed as the number of indexes vacuumed or cleaned up so far.
Author: Sami Imseih, based on suggestions by Nathan Bossart, Peter Geoghegan and Masahiko Sawada
Reviewed by: Nathan Bossart, Masahiko Sawada
Discussion: https://www.postgresql.org/message-id/flat/[email protected]
---
contrib/bloom/blvacuum.c | 6 +++
doc/src/sgml/monitoring.sgml | 21 +++++++++
src/backend/access/brin/brin.c | 9 ++--
src/backend/access/gin/ginvacuum.c | 9 ++++
src/backend/access/gist/gistvacuum.c | 5 +++
src/backend/access/hash/hash.c | 8 +++-
src/backend/access/hash/hashpage.c | 4 +-
src/backend/access/heap/vacuumlazy.c | 40 ++++++++++++++++-
src/backend/access/nbtree/nbtree.c | 3 ++
src/backend/access/spgist/spgvacuum.c | 3 ++
src/backend/access/transam/parallel.c | 5 +++
src/backend/catalog/index.c | 1 +
src/backend/catalog/system_views.sql | 3 +-
src/backend/commands/vacuumparallel.c | 63 ++++++++++++++++++++++++++-
src/include/access/genam.h | 3 ++
src/include/access/hash.h | 3 +-
src/include/access/parallel.h | 2 +
src/include/commands/progress.h | 2 +
src/include/commands/vacuum.h | 2 +
src/test/regress/expected/rules.out | 4 +-
20 files changed, 184 insertions(+), 12 deletions(-)
10.5% doc/src/sgml/
5.8% src/backend/access/brin/
4.4% src/backend/access/gin/
6.2% src/backend/access/hash/
17.7% src/backend/access/heap/
7.5% src/backend/access/
32.8% src/backend/commands/
6.1% src/include/access/
5.6% src/
diff --git a/contrib/bloom/blvacuum.c b/contrib/bloom/blvacuum.c
index 91fae5b0c0..19ae819cfe 100644
--- a/contrib/bloom/blvacuum.c
+++ b/contrib/bloom/blvacuum.c
@@ -136,6 +136,9 @@ blbulkdelete(IndexVacuumInfo *info, IndexBulkDeleteResult *stats,
GenericXLogAbort(gxlogState);
}
UnlockReleaseBuffer(buffer);
+
+ if (info->report_parallel_progress)
+ info->parallel_progress_callback(info->parallel_progress_arg);
}
/*
@@ -209,6 +212,9 @@ blvacuumcleanup(IndexVacuumInfo *info, IndexBulkDeleteResult *stats)
}
UnlockReleaseBuffer(buffer);
+
+ if (info->report_parallel_progress)
+ info->parallel_progress_callback(info->parallel_progress_arg);
}
IndexFreeSpaceMapVacuum(info->index);
diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml
index e5d622d514..c61443f3d5 100644
--- a/doc/src/sgml/monitoring.sgml
+++ b/doc/src/sgml/monitoring.sgml
@@ -6455,6 +6455,27 @@ FROM pg_stat_get_backend_idset() AS backendid;
Number of dead tuples collected since the last index vacuum cycle.
</para></entry>
</row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>indexes_total</structfield> <type>bigint</type>
+ </para>
+ <para>
+ Number of indexes that wil be vacuumed. This value will be
+ <literal>0</literal> if there are no indexes to vacuum or
+ vacuum failsafe is triggered. See <xref linkend="guc-vacuum-failsafe-age"/>
+ for more on vacuum failsafe.
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>indexes_completed</structfield> <type>bigint</type>
+ </para>
+ <para>
+ Number of indexes vacuumed in the current vacuum cycle.
+ </para></entry>
+ </row>
</tbody>
</tgroup>
</table>
diff --git a/src/backend/access/brin/brin.c b/src/backend/access/brin/brin.c
index 20b7d65b94..bd96ecf41e 100644
--- a/src/backend/access/brin/brin.c
+++ b/src/backend/access/brin/brin.c
@@ -78,7 +78,7 @@ static void brinsummarize(Relation index, Relation heapRel, BlockNumber pageRang
static void form_and_insert_tuple(BrinBuildState *state);
static void union_tuples(BrinDesc *bdesc, BrinMemTuple *a,
BrinTuple *b);
-static void brin_vacuum_scan(Relation idxrel, BufferAccessStrategy strategy);
+static void brin_vacuum_scan(Relation idxrel, BufferAccessStrategy strategy, IndexVacuumInfo *info);
static bool add_values_to_range(Relation idxRel, BrinDesc *bdesc,
BrinMemTuple *dtup, Datum *values, bool *nulls);
static bool check_null_keys(BrinValues *bval, ScanKey *nullkeys, int nnullkeys);
@@ -952,7 +952,7 @@ brinvacuumcleanup(IndexVacuumInfo *info, IndexBulkDeleteResult *stats)
heapRel = table_open(IndexGetRelation(RelationGetRelid(info->index), false),
AccessShareLock);
- brin_vacuum_scan(info->index, info->strategy);
+ brin_vacuum_scan(info->index, info->strategy, info);
brinsummarize(info->index, heapRel, BRIN_ALL_BLOCKRANGES, false,
&stats->num_index_tuples, &stats->num_index_tuples);
@@ -1659,7 +1659,7 @@ union_tuples(BrinDesc *bdesc, BrinMemTuple *a, BrinTuple *b)
* and such.
*/
static void
-brin_vacuum_scan(Relation idxrel, BufferAccessStrategy strategy)
+brin_vacuum_scan(Relation idxrel, BufferAccessStrategy strategy, IndexVacuumInfo *info)
{
BlockNumber nblocks;
BlockNumber blkno;
@@ -1681,6 +1681,9 @@ brin_vacuum_scan(Relation idxrel, BufferAccessStrategy strategy)
brin_page_cleanup(idxrel, buf);
ReleaseBuffer(buf);
+
+ if (info->report_parallel_progress)
+ info->parallel_progress_callback(info->parallel_progress_arg);
}
/*
diff --git a/src/backend/access/gin/ginvacuum.c b/src/backend/access/gin/ginvacuum.c
index b4fa5f6bf8..3d5e4600dc 100644
--- a/src/backend/access/gin/ginvacuum.c
+++ b/src/backend/access/gin/ginvacuum.c
@@ -633,6 +633,9 @@ ginbulkdelete(IndexVacuumInfo *info, IndexBulkDeleteResult *stats,
UnlockReleaseBuffer(buffer);
buffer = ReadBufferExtended(index, MAIN_FORKNUM, blkno,
RBM_NORMAL, info->strategy);
+
+ if (info->report_parallel_progress)
+ info->parallel_progress_callback(info->parallel_progress_arg);
}
/* right now we found leftmost page in entry's BTree */
@@ -677,6 +680,9 @@ ginbulkdelete(IndexVacuumInfo *info, IndexBulkDeleteResult *stats,
buffer = ReadBufferExtended(index, MAIN_FORKNUM, blkno,
RBM_NORMAL, info->strategy);
LockBuffer(buffer, GIN_EXCLUSIVE);
+
+ if (info->report_parallel_progress)
+ info->parallel_progress_callback(info->parallel_progress_arg);
}
MemoryContextDelete(gvs.tmpCxt);
@@ -775,6 +781,9 @@ ginvacuumcleanup(IndexVacuumInfo *info, IndexBulkDeleteResult *stats)
}
UnlockReleaseBuffer(buffer);
+
+ if (info->report_parallel_progress)
+ info->parallel_progress_callback(info->parallel_progress_arg);
}
/* Update the metapage with accurate page and entry counts */
diff --git a/src/backend/access/gist/gistvacuum.c b/src/backend/access/gist/gistvacuum.c
index 0aa6e58a62..da3a0540d0 100644
--- a/src/backend/access/gist/gistvacuum.c
+++ b/src/backend/access/gist/gistvacuum.c
@@ -223,7 +223,12 @@ gistvacuumscan(IndexVacuumInfo *info, IndexBulkDeleteResult *stats,
break;
/* Iterate over pages, then loop back to recheck length */
for (; blkno < num_pages; blkno++)
+ {
gistvacuumpage(&vstate, blkno, blkno);
+
+ if (info->report_parallel_progress)
+ info->parallel_progress_callback(info->parallel_progress_arg);
+ }
}
/*
diff --git a/src/backend/access/hash/hash.c b/src/backend/access/hash/hash.c
index c361509d68..14790adf61 100644
--- a/src/backend/access/hash/hash.c
+++ b/src/backend/access/hash/hash.c
@@ -550,7 +550,7 @@ loop_top:
cachedmetap->hashm_highmask,
cachedmetap->hashm_lowmask, &tuples_removed,
&num_index_tuples, split_cleanup,
- callback, callback_state);
+ callback, callback_state, info);
_hash_dropbuf(rel, bucket_buf);
@@ -686,7 +686,8 @@ hashbucketcleanup(Relation rel, Bucket cur_bucket, Buffer bucket_buf,
uint32 maxbucket, uint32 highmask, uint32 lowmask,
double *tuples_removed, double *num_index_tuples,
bool split_cleanup,
- IndexBulkDeleteCallback callback, void *callback_state)
+ IndexBulkDeleteCallback callback, void *callback_state,
+ IndexVacuumInfo *info)
{
BlockNumber blkno;
Buffer buf;
@@ -775,6 +776,9 @@ hashbucketcleanup(Relation rel, Bucket cur_bucket, Buffer bucket_buf,
if (num_index_tuples)
*num_index_tuples += 1;
}
+
+ if (info && info->report_parallel_progress)
+ info->parallel_progress_callback(info->parallel_progress_arg);
}
/* retain the pin on primary bucket page till end of bucket scan */
diff --git a/src/backend/access/hash/hashpage.c b/src/backend/access/hash/hashpage.c
index d2edcd4617..e9ce1a1110 100644
--- a/src/backend/access/hash/hashpage.c
+++ b/src/backend/access/hash/hashpage.c
@@ -759,7 +759,7 @@ restart_expand:
hashbucketcleanup(rel, old_bucket, buf_oblkno, start_oblkno, NULL,
maxbucket, highmask, lowmask, NULL, NULL, true,
- NULL, NULL);
+ NULL, NULL, NULL);
_hash_dropbuf(rel, buf_oblkno);
@@ -1327,7 +1327,7 @@ _hash_splitbucket(Relation rel,
hashbucketcleanup(rel, obucket, bucket_obuf,
BufferGetBlockNumber(bucket_obuf), NULL,
maxbucket, highmask, lowmask, NULL, NULL, true,
- NULL, NULL);
+ NULL, NULL, NULL);
}
else
{
diff --git a/src/backend/access/heap/vacuumlazy.c b/src/backend/access/heap/vacuumlazy.c
index dfbe37472f..120a2b18d7 100644
--- a/src/backend/access/heap/vacuumlazy.c
+++ b/src/backend/access/heap/vacuumlazy.c
@@ -420,6 +420,10 @@ heap_vacuum_rel(Relation rel, VacuumParams *params,
vacrel->rel = rel;
vac_open_indexes(vacrel->rel, RowExclusiveLock, &vacrel->nindexes,
&vacrel->indrels);
+
+ /* report number of indexes to vacuum */
+ pgstat_progress_update_param(PROGRESS_VACUUM_INDEX_TOTAL, vacrel->nindexes);
+
if (instrument && vacrel->nindexes > 0)
{
/* Copy index names used by instrumentation (not error reporting) */
@@ -2341,6 +2345,12 @@ lazy_vacuum_all_indexes(LVRelState *vacrel)
lazy_vacuum_one_index(indrel, istat, vacrel->old_live_tuples,
vacrel);
+ /*
+ * Done vacuuming an index. Increment the indexes completed
+ */
+ pgstat_progress_update_param(PROGRESS_VACUUM_INDEX_COMPLETED,
+ idx + 1);
+
if (lazy_check_wraparound_failsafe(vacrel))
{
/* Wraparound emergency -- end current index scan */
@@ -2384,6 +2394,13 @@ lazy_vacuum_all_indexes(LVRelState *vacrel)
pgstat_progress_update_param(PROGRESS_VACUUM_NUM_INDEX_VACUUMS,
vacrel->num_index_scans);
+ /*
+ * Reset the indexes completed at this point.
+ * If we end up in another index vacuum cycle, we will
+ * start counting from the start.
+ */
+ pgstat_progress_update_param(PROGRESS_VACUUM_INDEX_COMPLETED, 0);
+
return allindexes;
}
@@ -2633,10 +2650,17 @@ lazy_check_wraparound_failsafe(LVRelState *vacrel)
{
vacrel->failsafe_active = true;
- /* Disable index vacuuming, index cleanup, and heap rel truncation */
+ /*
+ * Disable index vacuuming, index cleanup, and heap rel truncation
+ *
+ * Also, report to progress.h that we are no longer tracking
+ * index vacuum/cleanup.
+ */
vacrel->do_index_vacuuming = false;
vacrel->do_index_cleanup = false;
vacrel->do_rel_truncate = false;
+ pgstat_progress_update_param(PROGRESS_VACUUM_INDEX_TOTAL, 0);
+ pgstat_progress_update_param(PROGRESS_VACUUM_INDEX_COMPLETED, 0);
ereport(WARNING,
(errmsg("bypassing nonessential maintenance of table \"%s.%s.%s\" as a failsafe after %d index scans",
@@ -2684,6 +2708,12 @@ lazy_cleanup_all_indexes(LVRelState *vacrel)
vacrel->indstats[idx] =
lazy_cleanup_one_index(indrel, istat, reltuples,
estimated_count, vacrel);
+
+ /*
+ * Done cleaning an index. Increment the indexes completed
+ */
+ pgstat_progress_update_param(PROGRESS_VACUUM_INDEX_COMPLETED,
+ idx + 1);
}
}
else
@@ -2693,6 +2723,12 @@ lazy_cleanup_all_indexes(LVRelState *vacrel)
vacrel->num_index_scans,
estimated_count);
}
+
+ /*
+ * Reset the indexes completed at this point.
+ */
+ pgstat_progress_update_param(PROGRESS_VACUUM_INDEX_COMPLETED, 0);
+
}
/*
@@ -2718,6 +2754,7 @@ lazy_vacuum_one_index(Relation indrel, IndexBulkDeleteResult *istat,
ivinfo.index = indrel;
ivinfo.analyze_only = false;
ivinfo.report_progress = false;
+ ivinfo.report_parallel_progress = false;
ivinfo.estimated_count = true;
ivinfo.message_level = DEBUG2;
ivinfo.num_heap_tuples = reltuples;
@@ -2766,6 +2803,7 @@ lazy_cleanup_one_index(Relation indrel, IndexBulkDeleteResult *istat,
ivinfo.index = indrel;
ivinfo.analyze_only = false;
ivinfo.report_progress = false;
+ ivinfo.report_parallel_progress = false;
ivinfo.estimated_count = estimated_count;
ivinfo.message_level = DEBUG2;
diff --git a/src/backend/access/nbtree/nbtree.c b/src/backend/access/nbtree/nbtree.c
index b52eca8f38..9904a9eb0c 100644
--- a/src/backend/access/nbtree/nbtree.c
+++ b/src/backend/access/nbtree/nbtree.c
@@ -998,6 +998,9 @@ btvacuumscan(IndexVacuumInfo *info, IndexBulkDeleteResult *stats,
if (info->report_progress)
pgstat_progress_update_param(PROGRESS_SCAN_BLOCKS_DONE,
scanblkno);
+
+ if (info->report_parallel_progress)
+ info->parallel_progress_callback(info->parallel_progress_arg);
}
}
diff --git a/src/backend/access/spgist/spgvacuum.c b/src/backend/access/spgist/spgvacuum.c
index 0049630532..b1ab491980 100644
--- a/src/backend/access/spgist/spgvacuum.c
+++ b/src/backend/access/spgist/spgvacuum.c
@@ -843,6 +843,9 @@ spgvacuumscan(spgBulkDeleteState *bds)
/* empty the pending-list after each page */
if (bds->pendingList != NULL)
spgprocesspending(bds);
+
+ if (bds->info->report_parallel_progress)
+ bds->info->parallel_progress_callback(bds->info->parallel_progress_arg);
}
}
diff --git a/src/backend/access/transam/parallel.c b/src/backend/access/transam/parallel.c
index ee0985c7ed..0b4bde8297 100644
--- a/src/backend/access/transam/parallel.c
+++ b/src/backend/access/transam/parallel.c
@@ -185,6 +185,8 @@ CreateParallelContext(const char *library_name, const char *function_name,
pcxt->library_name = pstrdup(library_name);
pcxt->function_name = pstrdup(function_name);
pcxt->error_context_stack = error_context_stack;
+ pcxt->parallel_progress_callback = NULL;
+ pcxt->parallel_progress_arg = NULL;
shm_toc_initialize_estimator(&pcxt->estimator);
dlist_push_head(&pcxt_list, &pcxt->node);
@@ -785,6 +787,9 @@ WaitForParallelWorkersToFinish(ParallelContext *pcxt)
*/
CHECK_FOR_INTERRUPTS();
+ if (pcxt->parallel_progress_callback)
+ pcxt->parallel_progress_callback(pcxt->parallel_progress_arg);
+
for (i = 0; i < pcxt->nworkers_launched; ++i)
{
/*
diff --git a/src/backend/catalog/index.c b/src/backend/catalog/index.c
index 61f1d3926a..11b32129a7 100644
--- a/src/backend/catalog/index.c
+++ b/src/backend/catalog/index.c
@@ -3348,6 +3348,7 @@ validate_index(Oid heapId, Oid indexId, Snapshot snapshot)
ivinfo.index = indexRelation;
ivinfo.analyze_only = false;
ivinfo.report_progress = true;
+ ivinfo.report_parallel_progress = false;
ivinfo.estimated_count = true;
ivinfo.message_level = DEBUG2;
ivinfo.num_heap_tuples = heapRelation->rd_rel->reltuples;
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index 2d8104b090..c37b20b91b 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1165,7 +1165,8 @@ CREATE VIEW pg_stat_progress_vacuum AS
END AS phase,
S.param2 AS heap_blks_total, S.param3 AS heap_blks_scanned,
S.param4 AS heap_blks_vacuumed, S.param5 AS index_vacuum_count,
- S.param6 AS max_dead_tuples, S.param7 AS num_dead_tuples
+ S.param6 AS max_dead_tuples, S.param7 AS num_dead_tuples,
+ S.param8 AS indexes_total, S.param9 AS indexes_completed
FROM pg_stat_get_progress_info('VACUUM') AS S
LEFT JOIN pg_database D ON S.datid = D.oid;
diff --git a/src/backend/commands/vacuumparallel.c b/src/backend/commands/vacuumparallel.c
index f26d796e52..85e2de5d54 100644
--- a/src/backend/commands/vacuumparallel.c
+++ b/src/backend/commands/vacuumparallel.c
@@ -30,6 +30,7 @@
#include "access/table.h"
#include "access/xact.h"
#include "catalog/index.h"
+#include "commands/progress.h"
#include "commands/vacuum.h"
#include "optimizer/paths.h"
#include "pgstat.h"
@@ -103,6 +104,17 @@ typedef struct PVShared
/* Counter for vacuuming and cleanup */
pg_atomic_uint32 idx;
+
+ /*
+ * Counter for vacuuming and cleanup progress reporting.
+ * This value is used to report index vacuum/cleanup progress
+ * in parallel_vacuum_progress_report. We keep this
+ * counter to avoid having to loop through
+ * ParallelVacuumState->indstats to determine the number
+ * of indexes completed.
+ */
+ pg_atomic_uint32 idx_completed_progress;
+
} PVShared;
/* Status used during parallel index vacuum or cleanup */
@@ -273,6 +285,9 @@ parallel_vacuum_init(Relation rel, Relation *indrels, int nindexes,
Assert(pcxt->nworkers > 0);
pvs->pcxt = pcxt;
+ pvs->pcxt->parallel_progress_callback = parallel_vacuum_progress_report;
+ pvs->pcxt->parallel_progress_arg = pvs;
+
/* Estimate size for index vacuum stats -- PARALLEL_VACUUM_KEY_INDEX_STATS */
est_indstats_len = mul_size(sizeof(PVIndStats), nindexes);
shm_toc_estimate_chunk(&pcxt->estimator, est_indstats_len);
@@ -364,6 +379,7 @@ parallel_vacuum_init(Relation rel, Relation *indrels, int nindexes,
pg_atomic_init_u32(&(shared->cost_balance), 0);
pg_atomic_init_u32(&(shared->active_nworkers), 0);
pg_atomic_init_u32(&(shared->idx), 0);
+ pg_atomic_init_u32(&(shared->idx_completed_progress), 0);
shm_toc_insert(pcxt->toc, PARALLEL_VACUUM_KEY_SHARED, shared);
pvs->shared = shared;
@@ -618,8 +634,9 @@ parallel_vacuum_process_all_indexes(ParallelVacuumState *pvs, int num_index_scan
vacuum));
}
- /* Reset the parallel index processing counter */
+ /* Reset the parallel index processing counter ( index progress counter also ) */
pg_atomic_write_u32(&(pvs->shared->idx), 0);
+ pg_atomic_write_u32(&(pvs->shared->idx_completed_progress), 0);
/* Setup the shared cost-based vacuum delay and launch workers */
if (nworkers > 0)
@@ -834,10 +851,13 @@ parallel_vacuum_process_one_index(ParallelVacuumState *pvs, Relation indrel,
ivinfo.index = indrel;
ivinfo.analyze_only = false;
ivinfo.report_progress = false;
+ ivinfo.report_parallel_progress = true;
ivinfo.message_level = DEBUG2;
ivinfo.estimated_count = pvs->shared->estimated_count;
ivinfo.num_heap_tuples = pvs->shared->reltuples;
ivinfo.strategy = pvs->bstrategy;
+ ivinfo.parallel_progress_callback = parallel_vacuum_progress_report;
+ ivinfo.parallel_progress_arg = pvs;
/* Update error traceback information */
pvs->indname = pstrdup(RelationGetRelationName(indrel));
@@ -846,9 +866,11 @@ parallel_vacuum_process_one_index(ParallelVacuumState *pvs, Relation indrel,
switch (indstats->status)
{
case PARALLEL_INDVAC_STATUS_NEED_BULKDELETE:
+ pgstat_progress_update_param(PROGRESS_VACUUM_PHASE, PROGRESS_VACUUM_PHASE_VACUUM_INDEX);
istat_res = vac_bulkdel_one_index(&ivinfo, istat, pvs->dead_items);
break;
case PARALLEL_INDVAC_STATUS_NEED_CLEANUP:
+ pgstat_progress_update_param(PROGRESS_VACUUM_PHASE, PROGRESS_VACUUM_PHASE_INDEX_CLEANUP);
istat_res = vac_cleanup_one_index(&ivinfo, istat);
break;
default:
@@ -888,6 +910,11 @@ parallel_vacuum_process_one_index(ParallelVacuumState *pvs, Relation indrel,
pvs->status = PARALLEL_INDVAC_STATUS_COMPLETED;
pfree(pvs->indname);
pvs->indname = NULL;
+
+ /*
+ * Update the number of indexes completed.
+ */
+ pg_atomic_add_fetch_u32(&(pvs->shared->idx_completed_progress), 1);
}
/*
@@ -1072,3 +1099,37 @@ parallel_vacuum_error_callback(void *arg)
return;
}
}
+
+/*
+ * Read the number of indexes vacuumed from the shared counter
+ * and report it to progress.h
+ *
+ * This function is only called by the leader process of
+ * a parallel vacuum.
+ *
+ * If report_parallel_progress is set to true,
+ * this function is called from major loops inside the
+ * ambulkdelete and amvacuumcleanup.
+ *
+ * This function is also called inside WaitForParallelWorkersToFinish
+ * when report_parallel_progress is set to true.
+ *
+ * The reason for calling this function in both vacuum AM's
+ * and WaitForParallelWorkersToFinish is to ensure that
+ * parallel vacuum progress is constantly being reported
+ * if the leader process is either waiting for parallel
+ * proceses to finish (WaitForParallelWorkersToFinish) or
+ * the leader process is the last to finish and still
+ * inside the vacuum AM's.
+ */
+void
+parallel_vacuum_progress_report(void *arg)
+{
+ ParallelVacuumState *pvs = (ParallelVacuumState *) arg;
+
+ if (IsParallelWorker())
+ return;
+
+ pgstat_progress_update_param(PROGRESS_VACUUM_INDEX_COMPLETED,
+ pg_atomic_read_u32(&(pvs->shared->idx_completed_progress)));
+}
diff --git a/src/include/access/genam.h b/src/include/access/genam.h
index e1c4fdbd03..6d5fc189f4 100644
--- a/src/include/access/genam.h
+++ b/src/include/access/genam.h
@@ -46,10 +46,13 @@ typedef struct IndexVacuumInfo
Relation index; /* the index being vacuumed */
bool analyze_only; /* ANALYZE (without any actual vacuum) */
bool report_progress; /* emit progress.h status reports */
+ bool report_parallel_progress; /* emit progress.h status reports for parallel vacuum */
bool estimated_count; /* num_heap_tuples is an estimate */
int message_level; /* ereport level for progress messages */
double num_heap_tuples; /* tuples remaining in heap */
BufferAccessStrategy strategy; /* access strategy for reads */
+ void (*parallel_progress_callback)(void *arg);
+ void *parallel_progress_arg;
} IndexVacuumInfo;
/*
diff --git a/src/include/access/hash.h b/src/include/access/hash.h
index da372841c4..14fb75a4ad 100644
--- a/src/include/access/hash.h
+++ b/src/include/access/hash.h
@@ -480,6 +480,7 @@ extern void hashbucketcleanup(Relation rel, Bucket cur_bucket,
uint32 maxbucket, uint32 highmask, uint32 lowmask,
double *tuples_removed, double *num_index_tuples,
bool split_cleanup,
- IndexBulkDeleteCallback callback, void *callback_state);
+ IndexBulkDeleteCallback callback, void *callback_state,
+ IndexVacuumInfo *info);
#endif /* HASH_H */
diff --git a/src/include/access/parallel.h b/src/include/access/parallel.h
index 1ec8e33af4..77e9c0a517 100644
--- a/src/include/access/parallel.h
+++ b/src/include/access/parallel.h
@@ -46,6 +46,8 @@ typedef struct ParallelContext
ParallelWorkerInfo *worker;
int nknown_attached_workers;
bool *known_attached_workers;
+ void (*parallel_progress_callback)(void *arg);
+ void *parallel_progress_arg;
} ParallelContext;
typedef struct ParallelWorkerContext
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index a28938caf4..0e97c6d4ef 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -25,6 +25,8 @@
#define PROGRESS_VACUUM_NUM_INDEX_VACUUMS 4
#define PROGRESS_VACUUM_MAX_DEAD_TUPLES 5
#define PROGRESS_VACUUM_NUM_DEAD_TUPLES 6
+#define PROGRESS_VACUUM_INDEX_TOTAL 7
+#define PROGRESS_VACUUM_INDEX_COMPLETED 8
/* Phases of vacuum (as advertised via PROGRESS_VACUUM_PHASE) */
#define PROGRESS_VACUUM_PHASE_SCAN_HEAP 1
diff --git a/src/include/commands/vacuum.h b/src/include/commands/vacuum.h
index 5d816ba7f4..9510d028d3 100644
--- a/src/include/commands/vacuum.h
+++ b/src/include/commands/vacuum.h
@@ -338,4 +338,6 @@ extern double anl_random_fract(void);
extern double anl_init_selection_state(int n);
extern double anl_get_next_S(double t, int n, double *stateptr);
+extern void parallel_vacuum_progress_report(void *arg);
+
#endif /* VACUUM_H */
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index 624d0e5aae..e0bf81247f 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -2018,7 +2018,9 @@ pg_stat_progress_vacuum| SELECT s.pid,
s.param4 AS heap_blks_vacuumed,
s.param5 AS index_vacuum_count,
s.param6 AS max_dead_tuples,
- s.param7 AS num_dead_tuples
+ s.param7 AS num_dead_tuples,
+ s.param8 AS indexes_total,
+ s.param9 AS indexes_completed
FROM (pg_stat_get_progress_info('VACUUM'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
LEFT JOIN pg_database d ON ((s.datid = d.oid)));
pg_stat_recovery_prefetch| SELECT s.stats_reset,
--
2.32.1 (Apple Git-133)
^ permalink raw reply [nested|flat] 44+ messages in thread
* Re: Add index scan progress to pg_stat_progress_vacuum
@ 2022-11-08 07:49 Masahiko Sawada <[email protected]>
parent: Imseih (AWS), Sami <[email protected]>
2 siblings, 0 replies; 44+ messages in thread
From: Masahiko Sawada @ 2022-11-08 07:49 UTC (permalink / raw)
To: Imseih (AWS), Sami <[email protected]>; +Cc: Robert Haas <[email protected]>; Bossart, Nathan <[email protected]>; Andres Freund <[email protected]>; Peter Geoghegan <[email protected]>; Justin Pryzby <[email protected]>; pgsql-hackers
On Thu, Nov 3, 2022 at 1:52 AM Imseih (AWS), Sami <[email protected]> wrote:
>
> Attached is v13-0001--Show-progress-for-index-vacuums.patch which addresses
> the latest comments.
Thank you for updating the patch!
> 4/ Went back to calling parallel_vacuum_progress_report inside
> WaitForParallelWorkersToFinish to cover the case when a
> leader is waiting for parallel workers to finish.
I don't think we need to modify WaitForParallelWorkersToFinish to
cover that case. Instead, I think the leader process can execute a new
function. The function will be like WaitForParallelWorkersToFinish()
but simpler; it just updates the progress information if necessary and
then checks if idx_completed_progress is equal to the number of
indexes to vacuum. If yes, return from the function and call
WaitForParallelWorkersToFinish() to wait for all workers to finish.
Otherwise, it naps by using WaitLatch() and does this loop again.
---
@@ -46,6 +46,8 @@ typedef struct ParallelContext
ParallelWorkerInfo *worker;
int nknown_attached_workers;
bool *known_attached_workers;
+ void (*parallel_progress_callback)(void *arg);
+ void *parallel_progress_arg;
} ParallelContext;
With the above change I suggested, I think we won't need to have a
callback function in ParallelContext. Instead, I think we can have
index-AMs call parallel_vacuum_report() if report_parallel_progress is
true.
Regards,
--
Masahiko Sawada
Amazon Web Services: https://aws.amazon.com
^ permalink raw reply [nested|flat] 44+ messages in thread
* Re: Add index scan progress to pg_stat_progress_vacuum
@ 2022-11-09 03:00 Andres Freund <[email protected]>
parent: Imseih (AWS), Sami <[email protected]>
0 siblings, 1 reply; 44+ messages in thread
From: Andres Freund @ 2022-11-09 03:00 UTC (permalink / raw)
To: Imseih (AWS), Sami <[email protected]>; +Cc: Masahiko Sawada <[email protected]>; Robert Haas <[email protected]>; Bossart, Nathan <[email protected]>; Peter Geoghegan <[email protected]>; Justin Pryzby <[email protected]>; pgsql-hackers
Hi,
On 2022-11-04 13:27:34 +0000, Imseih (AWS), Sami wrote:
> diff --git a/src/backend/access/gin/ginvacuum.c b/src/backend/access/gin/ginvacuum.c
> index b4fa5f6bf8..3d5e4600dc 100644
> --- a/src/backend/access/gin/ginvacuum.c
> +++ b/src/backend/access/gin/ginvacuum.c
> @@ -633,6 +633,9 @@ ginbulkdelete(IndexVacuumInfo *info, IndexBulkDeleteResult *stats,
> UnlockReleaseBuffer(buffer);
> buffer = ReadBufferExtended(index, MAIN_FORKNUM, blkno,
> RBM_NORMAL, info->strategy);
> +
> + if (info->report_parallel_progress)
> + info->parallel_progress_callback(info->parallel_progress_arg);
> }
>
> /* right now we found leftmost page in entry's BTree */
I don't think any of these progress callbacks should be done while pinning a
buffer and ...
> @@ -677,6 +680,9 @@ ginbulkdelete(IndexVacuumInfo *info, IndexBulkDeleteResult *stats,
> buffer = ReadBufferExtended(index, MAIN_FORKNUM, blkno,
> RBM_NORMAL, info->strategy);
> LockBuffer(buffer, GIN_EXCLUSIVE);
> +
> + if (info->report_parallel_progress)
> + info->parallel_progress_callback(info->parallel_progress_arg);
> }
>
> MemoryContextDelete(gvs.tmpCxt);
... definitely not while holding a buffer lock.
I also don't understand why info->parallel_progress_callback exists? It's only
set to parallel_vacuum_progress_report(). Why make this stuff more expensive
than it has to already be?
> +parallel_vacuum_progress_report(void *arg)
> +{
> + ParallelVacuumState *pvs = (ParallelVacuumState *) arg;
> +
> + if (IsParallelWorker())
> + return;
> +
> + pgstat_progress_update_param(PROGRESS_VACUUM_INDEX_COMPLETED,
> + pg_atomic_read_u32(&(pvs->shared->idx_completed_progress)));
> +}
So each of the places that call this need to make an additional external
function call for each page, just to find that there's nothing to do or to
make yet another indirect function call. This should probably a static inline
function.
This is called, for every single page, just to read the number of indexes
completed by workers? A number that barely ever changes?
This seems all wrong.
Greetings,
Andres Freund
^ permalink raw reply [nested|flat] 44+ messages in thread
* Re: Add index scan progress to pg_stat_progress_vacuum
@ 2022-11-11 19:10 Imseih (AWS), Sami <[email protected]>
parent: Andres Freund <[email protected]>
0 siblings, 1 reply; 44+ messages in thread
From: Imseih (AWS), Sami @ 2022-11-11 19:10 UTC (permalink / raw)
To: Andres Freund <[email protected]>; Masahiko Sawada <[email protected]>; +Cc: Robert Haas <[email protected]>; Bossart, Nathan <[email protected]>; Peter Geoghegan <[email protected]>; Justin Pryzby <[email protected]>; pgsql-hackers
> I don't think any of these progress callbacks should be done while pinning a
> buffer and ...
Good point.
> I also don't understand why info->parallel_progress_callback exists? It's only
> set to parallel_vacuum_progress_report(). Why make this stuff more expensive
> than it has to already be?
Agree. Modified the patch to avoid the callback .
> So each of the places that call this need to make an additional external
> function call for each page, just to find that there's nothing to do or to
> make yet another indirect function call. This should probably a static inline
> function.
Even better to just remove a function call altogether.
> This is called, for every single page, just to read the number of indexes
> completed by workers? A number that barely ever changes?
I will take the initial suggestion by Sawada-san to update the progress
every 1GB of blocks scanned.
Also, It sems to me that we don't need to track progress in brin indexes,
Since it is rare, if ever, this type of index will go through very heavy
block scans. In fact, I noticed the vacuum AMs for brin don't invoke the
vacuum_delay_point at all.
The attached patch addresses the feedback.
Regards,
Sami Imseih
Amazon Web Services (AWS)
Attachments:
[application/octet-stream] v15-0001-Add-2-new-columns-to-pg_stat_progress_vacuum.-Th.patch (21.4K, ../../[email protected]/2-v15-0001-Add-2-new-columns-to-pg_stat_progress_vacuum.-Th.patch)
download | inline diff:
From c59171a3ecee410380e9f62bfc2d678243842a2c Mon Sep 17 00:00:00 2001
From: "Sami Imseih (AWS)"
<[email protected]>
Date: Fri, 11 Nov 2022 18:40:14 +0000
Subject: [PATCH 1/1] Add 2 new columns to pg_stat_progress_vacuum. The columns
are indexes_total as the total indexes to be vacuumed or cleaned and
indexes_processed as the number of indexes vacuumed or cleaned up so far.
Author: Sami Imseih, based on suggestions by Nathan Bossart, Peter Geoghegan and Masahiko Sawada
Reviewed by: Nathan Bossart, Masahiko Sawada
Discussion: https://www.postgresql.org/message-id/flat/[email protected]
---
contrib/bloom/blvacuum.c | 9 ++++
doc/src/sgml/monitoring.sgml | 21 +++++++++
src/backend/access/gin/ginvacuum.c | 10 ++++
src/backend/access/gist/gistvacuum.c | 7 +++
src/backend/access/hash/hash.c | 7 +++
src/backend/access/heap/vacuumlazy.c | 40 +++++++++++++++-
src/backend/access/nbtree/nbtree.c | 3 ++
src/backend/access/spgist/spgvacuum.c | 6 +++
src/backend/catalog/index.c | 2 +
src/backend/catalog/system_views.sql | 3 +-
src/backend/commands/vacuumparallel.c | 66 +++++++++++++++++++++++++--
src/include/access/genam.h | 2 +
src/include/commands/progress.h | 2 +
src/include/commands/vacuum.h | 6 +++
src/test/regress/expected/rules.out | 4 +-
15 files changed, 182 insertions(+), 6 deletions(-)
diff --git a/contrib/bloom/blvacuum.c b/contrib/bloom/blvacuum.c
index 91fae5b0c0..92a557518f 100644
--- a/contrib/bloom/blvacuum.c
+++ b/contrib/bloom/blvacuum.c
@@ -15,12 +15,14 @@
#include "access/genam.h"
#include "bloom.h"
#include "catalog/storage.h"
+#include "commands/progress.h"
#include "commands/vacuum.h"
#include "miscadmin.h"
#include "postmaster/autovacuum.h"
#include "storage/bufmgr.h"
#include "storage/indexfsm.h"
#include "storage/lmgr.h"
+#include "utils/backend_progress.h"
/*
@@ -62,6 +64,10 @@ blbulkdelete(IndexVacuumInfo *info, IndexBulkDeleteResult *stats,
*itupEnd;
vacuum_delay_point();
+ if (info->report_parallel_progress && (blkno % REPORT_PARALLEL_VACUUM_EVERY_PAGES) == 0)
+ pgstat_progress_update_param(PROGRESS_VACUUM_INDEX_COMPLETED,
+ pg_atomic_read_u32(&(info->idx_completed_progress)));
+
buffer = ReadBufferExtended(index, MAIN_FORKNUM, blkno,
RBM_NORMAL, info->strategy);
@@ -192,6 +198,9 @@ blvacuumcleanup(IndexVacuumInfo *info, IndexBulkDeleteResult *stats)
Page page;
vacuum_delay_point();
+ if (info->report_parallel_progress && (blkno % REPORT_PARALLEL_VACUUM_EVERY_PAGES) == 0)
+ pgstat_progress_update_param(PROGRESS_VACUUM_INDEX_COMPLETED,
+ pg_atomic_read_u32(&(info->idx_completed_progress)));
buffer = ReadBufferExtended(index, MAIN_FORKNUM, blkno,
RBM_NORMAL, info->strategy);
diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml
index e5d622d514..c61443f3d5 100644
--- a/doc/src/sgml/monitoring.sgml
+++ b/doc/src/sgml/monitoring.sgml
@@ -6455,6 +6455,27 @@ FROM pg_stat_get_backend_idset() AS backendid;
Number of dead tuples collected since the last index vacuum cycle.
</para></entry>
</row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>indexes_total</structfield> <type>bigint</type>
+ </para>
+ <para>
+ Number of indexes that wil be vacuumed. This value will be
+ <literal>0</literal> if there are no indexes to vacuum or
+ vacuum failsafe is triggered. See <xref linkend="guc-vacuum-failsafe-age"/>
+ for more on vacuum failsafe.
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>indexes_completed</structfield> <type>bigint</type>
+ </para>
+ <para>
+ Number of indexes vacuumed in the current vacuum cycle.
+ </para></entry>
+ </row>
</tbody>
</tgroup>
</table>
diff --git a/src/backend/access/gin/ginvacuum.c b/src/backend/access/gin/ginvacuum.c
index b4fa5f6bf8..623ee78afa 100644
--- a/src/backend/access/gin/ginvacuum.c
+++ b/src/backend/access/gin/ginvacuum.c
@@ -17,12 +17,14 @@
#include "access/gin_private.h"
#include "access/ginxlog.h"
#include "access/xloginsert.h"
+#include "commands/progress.h"
#include "commands/vacuum.h"
#include "miscadmin.h"
#include "postmaster/autovacuum.h"
#include "storage/indexfsm.h"
#include "storage/lmgr.h"
#include "storage/predicate.h"
+#include "utils/backend_progress.h"
#include "utils/memutils.h"
struct GinVacuumState
@@ -665,6 +667,10 @@ ginbulkdelete(IndexVacuumInfo *info, IndexBulkDeleteResult *stats,
vacuum_delay_point();
+ if (info->report_parallel_progress && (blkno % REPORT_PARALLEL_VACUUM_EVERY_PAGES) == 0)
+ pgstat_progress_update_param(PROGRESS_VACUUM_INDEX_COMPLETED,
+ pg_atomic_read_u32(&(info->idx_completed_progress)));
+
for (i = 0; i < nRoot; i++)
{
ginVacuumPostingTree(&gvs, rootOfPostingTree[i]);
@@ -751,6 +757,10 @@ ginvacuumcleanup(IndexVacuumInfo *info, IndexBulkDeleteResult *stats)
vacuum_delay_point();
+ if (info->report_parallel_progress && (blkno % REPORT_PARALLEL_VACUUM_EVERY_PAGES) == 0)
+ pgstat_progress_update_param(PROGRESS_VACUUM_INDEX_COMPLETED,
+ pg_atomic_read_u32(&(info->idx_completed_progress)));
+
buffer = ReadBufferExtended(index, MAIN_FORKNUM, blkno,
RBM_NORMAL, info->strategy);
LockBuffer(buffer, GIN_SHARE);
diff --git a/src/backend/access/gist/gistvacuum.c b/src/backend/access/gist/gistvacuum.c
index 0aa6e58a62..45902e7aa9 100644
--- a/src/backend/access/gist/gistvacuum.c
+++ b/src/backend/access/gist/gistvacuum.c
@@ -17,11 +17,13 @@
#include "access/genam.h"
#include "access/gist_private.h"
#include "access/transam.h"
+#include "commands/progress.h"
#include "commands/vacuum.h"
#include "lib/integerset.h"
#include "miscadmin.h"
#include "storage/indexfsm.h"
#include "storage/lmgr.h"
+#include "utils/backend_progress.h"
#include "utils/memutils.h"
/* Working state needed by gistbulkdelete */
@@ -223,7 +225,12 @@ gistvacuumscan(IndexVacuumInfo *info, IndexBulkDeleteResult *stats,
break;
/* Iterate over pages, then loop back to recheck length */
for (; blkno < num_pages; blkno++)
+ {
gistvacuumpage(&vstate, blkno, blkno);
+ if (info->report_parallel_progress && (blkno % REPORT_PARALLEL_VACUUM_EVERY_PAGES) == 0)
+ pgstat_progress_update_param(PROGRESS_VACUUM_INDEX_COMPLETED,
+ pg_atomic_read_u32(&(info->idx_completed_progress)));
+ }
}
/*
diff --git a/src/backend/access/hash/hash.c b/src/backend/access/hash/hash.c
index c361509d68..798c93b6cb 100644
--- a/src/backend/access/hash/hash.c
+++ b/src/backend/access/hash/hash.c
@@ -505,6 +505,13 @@ loop_top:
blkno = bucket_blkno;
+ /*
+ * For hash indexes, we report parallel vacuum progress
+ * for every bucket.
+ */
+ if (info->report_parallel_progress)
+ pgstat_progress_update_param(PROGRESS_VACUUM_INDEX_COMPLETED,
+ pg_atomic_read_u32(&(info->idx_completed_progress)));
/*
* We need to acquire a cleanup lock on the primary bucket page to out
* wait concurrent scans before deleting the dead tuples.
diff --git a/src/backend/access/heap/vacuumlazy.c b/src/backend/access/heap/vacuumlazy.c
index dfbe37472f..120a2b18d7 100644
--- a/src/backend/access/heap/vacuumlazy.c
+++ b/src/backend/access/heap/vacuumlazy.c
@@ -420,6 +420,10 @@ heap_vacuum_rel(Relation rel, VacuumParams *params,
vacrel->rel = rel;
vac_open_indexes(vacrel->rel, RowExclusiveLock, &vacrel->nindexes,
&vacrel->indrels);
+
+ /* report number of indexes to vacuum */
+ pgstat_progress_update_param(PROGRESS_VACUUM_INDEX_TOTAL, vacrel->nindexes);
+
if (instrument && vacrel->nindexes > 0)
{
/* Copy index names used by instrumentation (not error reporting) */
@@ -2341,6 +2345,12 @@ lazy_vacuum_all_indexes(LVRelState *vacrel)
lazy_vacuum_one_index(indrel, istat, vacrel->old_live_tuples,
vacrel);
+ /*
+ * Done vacuuming an index. Increment the indexes completed
+ */
+ pgstat_progress_update_param(PROGRESS_VACUUM_INDEX_COMPLETED,
+ idx + 1);
+
if (lazy_check_wraparound_failsafe(vacrel))
{
/* Wraparound emergency -- end current index scan */
@@ -2384,6 +2394,13 @@ lazy_vacuum_all_indexes(LVRelState *vacrel)
pgstat_progress_update_param(PROGRESS_VACUUM_NUM_INDEX_VACUUMS,
vacrel->num_index_scans);
+ /*
+ * Reset the indexes completed at this point.
+ * If we end up in another index vacuum cycle, we will
+ * start counting from the start.
+ */
+ pgstat_progress_update_param(PROGRESS_VACUUM_INDEX_COMPLETED, 0);
+
return allindexes;
}
@@ -2633,10 +2650,17 @@ lazy_check_wraparound_failsafe(LVRelState *vacrel)
{
vacrel->failsafe_active = true;
- /* Disable index vacuuming, index cleanup, and heap rel truncation */
+ /*
+ * Disable index vacuuming, index cleanup, and heap rel truncation
+ *
+ * Also, report to progress.h that we are no longer tracking
+ * index vacuum/cleanup.
+ */
vacrel->do_index_vacuuming = false;
vacrel->do_index_cleanup = false;
vacrel->do_rel_truncate = false;
+ pgstat_progress_update_param(PROGRESS_VACUUM_INDEX_TOTAL, 0);
+ pgstat_progress_update_param(PROGRESS_VACUUM_INDEX_COMPLETED, 0);
ereport(WARNING,
(errmsg("bypassing nonessential maintenance of table \"%s.%s.%s\" as a failsafe after %d index scans",
@@ -2684,6 +2708,12 @@ lazy_cleanup_all_indexes(LVRelState *vacrel)
vacrel->indstats[idx] =
lazy_cleanup_one_index(indrel, istat, reltuples,
estimated_count, vacrel);
+
+ /*
+ * Done cleaning an index. Increment the indexes completed
+ */
+ pgstat_progress_update_param(PROGRESS_VACUUM_INDEX_COMPLETED,
+ idx + 1);
}
}
else
@@ -2693,6 +2723,12 @@ lazy_cleanup_all_indexes(LVRelState *vacrel)
vacrel->num_index_scans,
estimated_count);
}
+
+ /*
+ * Reset the indexes completed at this point.
+ */
+ pgstat_progress_update_param(PROGRESS_VACUUM_INDEX_COMPLETED, 0);
+
}
/*
@@ -2718,6 +2754,7 @@ lazy_vacuum_one_index(Relation indrel, IndexBulkDeleteResult *istat,
ivinfo.index = indrel;
ivinfo.analyze_only = false;
ivinfo.report_progress = false;
+ ivinfo.report_parallel_progress = false;
ivinfo.estimated_count = true;
ivinfo.message_level = DEBUG2;
ivinfo.num_heap_tuples = reltuples;
@@ -2766,6 +2803,7 @@ lazy_cleanup_one_index(Relation indrel, IndexBulkDeleteResult *istat,
ivinfo.index = indrel;
ivinfo.analyze_only = false;
ivinfo.report_progress = false;
+ ivinfo.report_parallel_progress = false;
ivinfo.estimated_count = estimated_count;
ivinfo.message_level = DEBUG2;
diff --git a/src/backend/access/nbtree/nbtree.c b/src/backend/access/nbtree/nbtree.c
index b52eca8f38..374a712952 100644
--- a/src/backend/access/nbtree/nbtree.c
+++ b/src/backend/access/nbtree/nbtree.c
@@ -998,6 +998,9 @@ btvacuumscan(IndexVacuumInfo *info, IndexBulkDeleteResult *stats,
if (info->report_progress)
pgstat_progress_update_param(PROGRESS_SCAN_BLOCKS_DONE,
scanblkno);
+ if (info->report_parallel_progress && (scanblkno % REPORT_PARALLEL_VACUUM_EVERY_PAGES) == 0)
+ pgstat_progress_update_param(PROGRESS_VACUUM_INDEX_COMPLETED,
+ pg_atomic_read_u32(&(info->idx_completed_progress)));
}
}
diff --git a/src/backend/access/spgist/spgvacuum.c b/src/backend/access/spgist/spgvacuum.c
index 0049630532..1f13bc28ce 100644
--- a/src/backend/access/spgist/spgvacuum.c
+++ b/src/backend/access/spgist/spgvacuum.c
@@ -21,11 +21,13 @@
#include "access/transam.h"
#include "access/xloginsert.h"
#include "catalog/storage_xlog.h"
+#include "commands/progress.h"
#include "commands/vacuum.h"
#include "miscadmin.h"
#include "storage/bufmgr.h"
#include "storage/indexfsm.h"
#include "storage/lmgr.h"
+#include "utils/backend_progress.h"
#include "utils/snapmgr.h"
@@ -843,6 +845,10 @@ spgvacuumscan(spgBulkDeleteState *bds)
/* empty the pending-list after each page */
if (bds->pendingList != NULL)
spgprocesspending(bds);
+ /* report parallel vacuum progress */
+ if (bds->info->report_parallel_progress && (blkno % REPORT_PARALLEL_VACUUM_EVERY_PAGES) == 0)
+ pgstat_progress_update_param(PROGRESS_VACUUM_INDEX_COMPLETED,
+ pg_atomic_read_u32(&(bds->info->idx_completed_progress)));
}
}
diff --git a/src/backend/catalog/index.c b/src/backend/catalog/index.c
index 61f1d3926a..3728750569 100644
--- a/src/backend/catalog/index.c
+++ b/src/backend/catalog/index.c
@@ -3348,10 +3348,12 @@ validate_index(Oid heapId, Oid indexId, Snapshot snapshot)
ivinfo.index = indexRelation;
ivinfo.analyze_only = false;
ivinfo.report_progress = true;
+ ivinfo.report_parallel_progress = false;
ivinfo.estimated_count = true;
ivinfo.message_level = DEBUG2;
ivinfo.num_heap_tuples = heapRelation->rd_rel->reltuples;
ivinfo.strategy = NULL;
+ pg_atomic_init_u32(&(ivinfo.idx_completed_progress), 0);
/*
* Encode TIDs as int8 values for the sort, rather than directly sorting
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index 2d8104b090..c37b20b91b 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1165,7 +1165,8 @@ CREATE VIEW pg_stat_progress_vacuum AS
END AS phase,
S.param2 AS heap_blks_total, S.param3 AS heap_blks_scanned,
S.param4 AS heap_blks_vacuumed, S.param5 AS index_vacuum_count,
- S.param6 AS max_dead_tuples, S.param7 AS num_dead_tuples
+ S.param6 AS max_dead_tuples, S.param7 AS num_dead_tuples,
+ S.param8 AS indexes_total, S.param9 AS indexes_completed
FROM pg_stat_get_progress_info('VACUUM') AS S
LEFT JOIN pg_database D ON S.datid = D.oid;
diff --git a/src/backend/commands/vacuumparallel.c b/src/backend/commands/vacuumparallel.c
index f26d796e52..c3579af169 100644
--- a/src/backend/commands/vacuumparallel.c
+++ b/src/backend/commands/vacuumparallel.c
@@ -30,6 +30,7 @@
#include "access/table.h"
#include "access/xact.h"
#include "catalog/index.h"
+#include "commands/progress.h"
#include "commands/vacuum.h"
#include "optimizer/paths.h"
#include "pgstat.h"
@@ -103,6 +104,17 @@ typedef struct PVShared
/* Counter for vacuuming and cleanup */
pg_atomic_uint32 idx;
+
+ /*
+ * Counter for vacuuming and cleanup progress reporting.
+ * This value is used to report index vacuum/cleanup progress
+ * in parallel_vacuum_progress_report. We keep this
+ * counter to avoid having to loop through
+ * ParallelVacuumState->indstats to determine the number
+ * of indexes completed.
+ */
+ pg_atomic_uint32 idx_completed_progress;
+
} PVShared;
/* Status used during parallel index vacuum or cleanup */
@@ -213,6 +225,7 @@ static void parallel_vacuum_process_one_index(ParallelVacuumState *pvs, Relation
static bool parallel_vacuum_index_is_parallel_safe(Relation indrel, int num_index_scans,
bool vacuum);
static void parallel_vacuum_error_callback(void *arg);
+static void parallel_wait_for_workers_to_finish(ParallelVacuumState *pvs);
/*
* Try to enter parallel mode and create a parallel context. Then initialize
@@ -364,6 +377,7 @@ parallel_vacuum_init(Relation rel, Relation *indrels, int nindexes,
pg_atomic_init_u32(&(shared->cost_balance), 0);
pg_atomic_init_u32(&(shared->active_nworkers), 0);
pg_atomic_init_u32(&(shared->idx), 0);
+ pg_atomic_init_u32(&(shared->idx_completed_progress), 0);
shm_toc_insert(pcxt->toc, PARALLEL_VACUUM_KEY_SHARED, shared);
pvs->shared = shared;
@@ -618,8 +632,9 @@ parallel_vacuum_process_all_indexes(ParallelVacuumState *pvs, int num_index_scan
vacuum));
}
- /* Reset the parallel index processing counter */
+ /* Reset the parallel index processing counter ( index progress counter also ) */
pg_atomic_write_u32(&(pvs->shared->idx), 0);
+ pg_atomic_write_u32(&(pvs->shared->idx_completed_progress), 0);
/* Setup the shared cost-based vacuum delay and launch workers */
if (nworkers > 0)
@@ -688,7 +703,21 @@ parallel_vacuum_process_all_indexes(ParallelVacuumState *pvs, int num_index_scan
*/
if (nworkers > 0)
{
- /* Wait for all vacuum workers to finish */
+ /*
+ * To wait for parallel workers to finish,
+ * first call parallel_wait_for_workers_to_finish
+ * which is responsible for reporting the
+ * number of indexes completed.
+ *
+ * Afterwards, WaitForParallelWorkersToFinish is called
+ * to do the real work of waiting for parallel workers
+ * to finish.
+ *
+ * Note: Both routines will acquire a WaitLatch in their
+ * respective loops.
+ */
+ parallel_wait_for_workers_to_finish(pvs);
+
WaitForParallelWorkersToFinish(pvs->pcxt);
for (int i = 0; i < pvs->pcxt->nworkers_launched; i++)
@@ -838,7 +867,12 @@ parallel_vacuum_process_one_index(ParallelVacuumState *pvs, Relation indrel,
ivinfo.estimated_count = pvs->shared->estimated_count;
ivinfo.num_heap_tuples = pvs->shared->reltuples;
ivinfo.strategy = pvs->bstrategy;
-
+ ivinfo.idx_completed_progress = pvs->shared->idx_completed_progress;
+ /* Only the leader should report parallel vacuum progress */
+ if (!IsParallelWorker())
+ ivinfo.report_parallel_progress = true;
+ else
+ ivinfo.report_parallel_progress = false;
/* Update error traceback information */
pvs->indname = pstrdup(RelationGetRelationName(indrel));
pvs->status = indstats->status;
@@ -857,6 +891,9 @@ parallel_vacuum_process_one_index(ParallelVacuumState *pvs, Relation indrel,
RelationGetRelationName(indrel));
}
+ if (ivinfo.report_parallel_progress)
+ pgstat_progress_update_param(PROGRESS_VACUUM_INDEX_COMPLETED,
+ pg_atomic_read_u32(&(ivinfo.idx_completed_progress)));
/*
* Copy the index bulk-deletion result returned from ambulkdelete and
* amvacuumcleanup to the DSM segment if it's the first cycle because they
@@ -888,6 +925,9 @@ parallel_vacuum_process_one_index(ParallelVacuumState *pvs, Relation indrel,
pvs->status = PARALLEL_INDVAC_STATUS_COMPLETED;
pfree(pvs->indname);
pvs->indname = NULL;
+
+ /* Update the number of indexes completed. */
+ pg_atomic_add_fetch_u32(&(pvs->shared->idx_completed_progress), 1);
}
/*
@@ -1072,3 +1112,23 @@ parallel_vacuum_error_callback(void *arg)
return;
}
}
+
+/*
+ * Check if we are done vacuuming indexes and report
+ * progress.
+ *
+ * We nap using with a WaitLatch to avoid a busy loop.
+ */
+void
+parallel_wait_for_workers_to_finish(ParallelVacuumState *pvs)
+{
+ while (pg_atomic_read_u32(&(pvs->shared->idx_completed_progress)) < pvs->nindexes)
+ {
+ pgstat_progress_update_param(PROGRESS_VACUUM_INDEX_COMPLETED,
+ pg_atomic_read_u32(&(pvs->shared->idx_completed_progress)));
+
+ (void) WaitLatch(MyLatch, WL_LATCH_SET | WL_EXIT_ON_PM_DEATH, -1,
+ WAIT_EVENT_PARALLEL_FINISH);
+ ResetLatch(MyLatch);
+ }
+}
diff --git a/src/include/access/genam.h b/src/include/access/genam.h
index e1c4fdbd03..bf69c93d31 100644
--- a/src/include/access/genam.h
+++ b/src/include/access/genam.h
@@ -46,10 +46,12 @@ typedef struct IndexVacuumInfo
Relation index; /* the index being vacuumed */
bool analyze_only; /* ANALYZE (without any actual vacuum) */
bool report_progress; /* emit progress.h status reports */
+ bool report_parallel_progress; /* emit progress.h status reports for parallel vacuum */
bool estimated_count; /* num_heap_tuples is an estimate */
int message_level; /* ereport level for progress messages */
double num_heap_tuples; /* tuples remaining in heap */
BufferAccessStrategy strategy; /* access strategy for reads */
+ pg_atomic_uint32 idx_completed_progress;
} IndexVacuumInfo;
/*
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index a28938caf4..0e97c6d4ef 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -25,6 +25,8 @@
#define PROGRESS_VACUUM_NUM_INDEX_VACUUMS 4
#define PROGRESS_VACUUM_MAX_DEAD_TUPLES 5
#define PROGRESS_VACUUM_NUM_DEAD_TUPLES 6
+#define PROGRESS_VACUUM_INDEX_TOTAL 7
+#define PROGRESS_VACUUM_INDEX_COMPLETED 8
/* Phases of vacuum (as advertised via PROGRESS_VACUUM_PHASE) */
#define PROGRESS_VACUUM_PHASE_SCAN_HEAP 1
diff --git a/src/include/commands/vacuum.h b/src/include/commands/vacuum.h
index 5d816ba7f4..34cf36c425 100644
--- a/src/include/commands/vacuum.h
+++ b/src/include/commands/vacuum.h
@@ -64,6 +64,12 @@
/* value for checking vacuum flags */
#define VACUUM_OPTION_MAX_VALID_VALUE ((1 << 3) - 1)
+/*
+ * Parallel Index vacuum progress is reported every 1GB of blocks
+ * scanned.
+ */
+#define REPORT_PARALLEL_VACUUM_EVERY_PAGES ((BlockNumber) (((uint64) 1024 * 1024 * 1024) / BLCKSZ))
+
/* Abstract type for parallel vacuum state */
typedef struct ParallelVacuumState ParallelVacuumState;
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index 624d0e5aae..e0bf81247f 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -2018,7 +2018,9 @@ pg_stat_progress_vacuum| SELECT s.pid,
s.param4 AS heap_blks_vacuumed,
s.param5 AS index_vacuum_count,
s.param6 AS max_dead_tuples,
- s.param7 AS num_dead_tuples
+ s.param7 AS num_dead_tuples,
+ s.param8 AS indexes_total,
+ s.param9 AS indexes_completed
FROM (pg_stat_get_progress_info('VACUUM'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
LEFT JOIN pg_database d ON ((s.datid = d.oid)));
pg_stat_recovery_prefetch| SELECT s.stats_reset,
--
2.37.1
^ permalink raw reply [nested|flat] 44+ messages in thread
* Re: Add index scan progress to pg_stat_progress_vacuum
@ 2022-11-18 13:07 Masahiko Sawada <[email protected]>
parent: Imseih (AWS), Sami <[email protected]>
0 siblings, 1 reply; 44+ messages in thread
From: Masahiko Sawada @ 2022-11-18 13:07 UTC (permalink / raw)
To: Imseih (AWS), Sami <[email protected]>; +Cc: Andres Freund <[email protected]>; Robert Haas <[email protected]>; Bossart, Nathan <[email protected]>; Peter Geoghegan <[email protected]>; Justin Pryzby <[email protected]>; pgsql-hackers
On Sat, Nov 12, 2022 at 4:10 AM Imseih (AWS), Sami <[email protected]> wrote:
>
> > I don't think any of these progress callbacks should be done while pinning a
> > buffer and ...
>
> Good point.
>
> > I also don't understand why info->parallel_progress_callback exists? It's only
> > set to parallel_vacuum_progress_report(). Why make this stuff more expensive
> > than it has to already be?
>
> Agree. Modified the patch to avoid the callback .
>
> > So each of the places that call this need to make an additional external
> > function call for each page, just to find that there's nothing to do or to
> > make yet another indirect function call. This should probably a static inline
> > function.
>
> Even better to just remove a function call altogether.
>
> > This is called, for every single page, just to read the number of indexes
> > completed by workers? A number that barely ever changes?
>
> I will take the initial suggestion by Sawada-san to update the progress
> every 1GB of blocks scanned.
>
> Also, It sems to me that we don't need to track progress in brin indexes,
> Since it is rare, if ever, this type of index will go through very heavy
> block scans. In fact, I noticed the vacuum AMs for brin don't invoke the
> vacuum_delay_point at all.
>
> The attached patch addresses the feedback.
>
Thank you for updating the patch! Here are review comments on v15 patch:
+ <para>
+ Number of indexes that wil be vacuumed. This value will be
+ <literal>0</literal> if there are no indexes to vacuum or
+ vacuum failsafe is triggered.
I think that indexes_total should be 0 also when INDEX_CLEANUP is off.
---
+ /*
+ * Reset the indexes completed at this point.
+ * If we end up in another index vacuum cycle, we will
+ * start counting from the start.
+ */
+ pgstat_progress_update_param(PROGRESS_VACUUM_INDEX_COMPLETED, 0);
I think we don't need to reset it at the end of index vacuuming. There
is a small window before switching to the next phase. If we reset this
value while showing "index vacuuming" phase, the user might get
confused. Instead, we can reset it at the beginning of the index
vacuuming.
---
+void
+parallel_wait_for_workers_to_finish(ParallelVacuumState *pvs)
+{
+ while (pg_atomic_read_u32(&(pvs->shared->idx_completed_progress))
< pvs->nindexes)
+ {
+ pgstat_progress_update_param(PROGRESS_VACUUM_INDEX_COMPLETED,
+
pg_atomic_read_u32(&(pvs->shared->idx_completed_progress)));
+
+ (void) WaitLatch(MyLatch, WL_LATCH_SET |
WL_EXIT_ON_PM_DEATH, -1,
+ WAIT_EVENT_PARALLEL_FINISH);
+ ResetLatch(MyLatch);
+ }
+}
We should add CHECK_FOR_INTERRUPTS() at the beginning of the loop to
make the wait interruptible.
I think it would be better to update the counter only when the value
has been increased.
I think we should set a timeout, say 1 sec, to WaitLatch so that it
can periodically check the progress.
Probably it's better to have a new wait event for this wait in order
to distinguish wait for workers to complete index vacuum from the wait
for workers to exit.
---
@@ -838,7 +867,12 @@
parallel_vacuum_process_one_index(ParallelVacuumState *pvs, Relation
indrel,
ivinfo.estimated_count = pvs->shared->estimated_count;
ivinfo.num_heap_tuples = pvs->shared->reltuples;
ivinfo.strategy = pvs->bstrategy;
-
+ ivinfo.idx_completed_progress = pvs->shared->idx_completed_progress;
and
@@ -998,6 +998,9 @@ btvacuumscan(IndexVacuumInfo *info,
IndexBulkDeleteResult *stats,
if (info->report_progress)
pgstat_progress_update_param(PROGRESS_SCAN_BLOCKS_DONE,
scanblkno);
+ if (info->report_parallel_progress &&
(scanblkno % REPORT_PARALLEL_VACUUM_EVERY_PAGES) == 0)
+
pgstat_progress_update_param(PROGRESS_VACUUM_INDEX_COMPLETED,
+
pg_atomic_read_u32(&(info->idx_completed_progress)));
}
I think this doesn't work, since ivinfo.idx_completed is in the
backend-local memory. Instead, I think we can have a function in
vacuumparallel.c that updates the progress. Then we can have index AM
call this function.
---
+ if (!IsParallelWorker())
+ ivinfo.report_parallel_progress = true;
+ else
+ ivinfo.report_parallel_progress = false;
We can do like:
ivinfo.report_parallel_progress = !IsParallelWorker();
Regards,
--
Masahiko Sawada
Amazon Web Services: https://aws.amazon.com
^ permalink raw reply [nested|flat] 44+ messages in thread
* Re: Add index scan progress to pg_stat_progress_vacuum
@ 2022-11-28 23:57 Imseih (AWS), Sami <[email protected]>
parent: Masahiko Sawada <[email protected]>
0 siblings, 1 reply; 44+ messages in thread
From: Imseih (AWS), Sami @ 2022-11-28 23:57 UTC (permalink / raw)
To: Masahiko Sawada <[email protected]>; +Cc: Andres Freund <[email protected]>; Robert Haas <[email protected]>; Bossart, Nathan <[email protected]>; Peter Geoghegan <[email protected]>; Justin Pryzby <[email protected]>; pgsql-hackers
> I think that indexes_total should be 0 also when INDEX_CLEANUP is off.
Patch updated for handling of INDEX_CLEANUP = off, with an update to
the documentation as well.
> I think we don't need to reset it at the end of index vacuuming. There
> is a small window before switching to the next phase. If we reset this
> value while showing "index vacuuming" phase, the user might get
> confused. Instead, we can reset it at the beginning of the index
> vacuuming.
No, I think the way it's currently done is correct. We want to reset the number
of indexes completed before we increase the num_index_scans ( index vacuum cycle ).
This ensures that when we enter a new index cycle, the number of indexes completed
are already reset. The 2 fields that matter here is how many indexes are vacuumed in the
currently cycle and this is called out in the documentation as such.
> We should add CHECK_FOR_INTERRUPTS() at the beginning of the loop to
> make the wait interruptible.
Done.
> I think it would be better to update the counter only when the value
> has been increased.
Done. Did so by checking the progress value from the beentry directly
in the function. We can do a more generalized
> I think we should set a timeout, say 1 sec, to WaitLatch so that it
> can periodically check the progress.
Done.
> Probably it's better to have a new wait event for this wait in order
> to distinguish wait for workers to complete index vacuum from the wait
> for workers to exit.
I was trying to avoid introducing a new wait event, but thinking about it,
I agree with your suggestion.
I created a new ParallelVacuumFinish wait event and documentation
for the wait event.
> I think this doesn't work, since ivinfo.idx_completed is in the
> backend-local memory. Instead, I think we can have a function in
> vacuumparallel.c that updates the progress. Then we can have index AM
> call this function.
Yeah, you're absolutely correct.
To make this work correctly, I did something similar to VacuumActiveNWorkers
and introduced an extern variable called ParallelVacuumProgress.
This variable points to pvs->shared->idx_completed_progress.
The index AMs then call parallel_vacuum_update_progress which
Is responsible for updating the progress with the current value
of ParallelVacuumProgress.
ParallelVacuumProgress is also set to NULL at the end of every index cycle.
> ivinfo.report_parallel_progress = !IsParallelWorker();
Done
Regards,
Sami Imseih
Amazon Web Services (AWS)
Attachments:
[application/octet-stream] v16-0001-Add-2-new-columns-to-pg_stat_progress_vacuum.-Th.patch (25.4K, ../../[email protected]/2-v16-0001-Add-2-new-columns-to-pg_stat_progress_vacuum.-Th.patch)
download | inline diff:
From db03efc9f21fa8cd9597cff146bbcce0becdf86a Mon Sep 17 00:00:00 2001
From: "Imseih (AWS)" <[email protected]>
Date: Mon, 28 Nov 2022 17:47:04 -0600
Subject: [PATCH 1/1] Add 2 new columns to pg_stat_progress_vacuum. The columns
are indexes_total as the total indexes to be vacuumed or cleaned and
indexes_processed as the number of indexes vacuumed or cleaned up so far.
Author: Sami Imseih, based on suggestions by Nathan Bossart, Peter Geoghegan and Masahiko Sawada
Reviewed by: Nathan Bossart, Masahiko Sawada
Discussion: https://www.postgresql.org/message-id/flat/[email protected]
---
contrib/bloom/blvacuum.c | 7 ++
doc/src/sgml/monitoring.sgml | 26 ++++++
src/backend/access/gin/ginvacuum.c | 8 ++
src/backend/access/gist/gistvacuum.c | 6 ++
src/backend/access/hash/hash.c | 6 ++
src/backend/access/heap/vacuumlazy.c | 33 ++++++-
src/backend/access/nbtree/nbtree.c | 2 +
src/backend/access/spgist/spgvacuum.c | 5 ++
src/backend/catalog/index.c | 1 +
src/backend/catalog/system_views.sql | 3 +-
src/backend/commands/vacuum.c | 2 +
src/backend/commands/vacuumparallel.c | 110 +++++++++++++++++++++++-
src/backend/utils/activity/wait_event.c | 3 +
src/include/access/genam.h | 1 +
src/include/commands/progress.h | 2 +
src/include/commands/vacuum.h | 9 ++
src/include/utils/wait_event.h | 3 +-
src/test/regress/expected/rules.out | 4 +-
18 files changed, 223 insertions(+), 8 deletions(-)
diff --git a/contrib/bloom/blvacuum.c b/contrib/bloom/blvacuum.c
index 91fae5b..023fc7e 100644
--- a/contrib/bloom/blvacuum.c
+++ b/contrib/bloom/blvacuum.c
@@ -15,12 +15,14 @@
#include "access/genam.h"
#include "bloom.h"
#include "catalog/storage.h"
+#include "commands/progress.h"
#include "commands/vacuum.h"
#include "miscadmin.h"
#include "postmaster/autovacuum.h"
#include "storage/bufmgr.h"
#include "storage/indexfsm.h"
#include "storage/lmgr.h"
+#include "utils/backend_progress.h"
/*
@@ -62,6 +64,9 @@ blbulkdelete(IndexVacuumInfo *info, IndexBulkDeleteResult *stats,
*itupEnd;
vacuum_delay_point();
+ if (info->report_parallel_progress && (blkno % REPORT_PARALLEL_VACUUM_EVERY_PAGES) == 0)
+ parallel_vacuum_update_progress();
+
buffer = ReadBufferExtended(index, MAIN_FORKNUM, blkno,
RBM_NORMAL, info->strategy);
@@ -192,6 +197,8 @@ blvacuumcleanup(IndexVacuumInfo *info, IndexBulkDeleteResult *stats)
Page page;
vacuum_delay_point();
+ if (info->report_parallel_progress && (blkno % REPORT_PARALLEL_VACUUM_EVERY_PAGES) == 0)
+ parallel_vacuum_update_progress();
buffer = ReadBufferExtended(index, MAIN_FORKNUM, blkno,
RBM_NORMAL, info->strategy);
diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml
index 5579b8b..477dfe2 100644
--- a/doc/src/sgml/monitoring.sgml
+++ b/doc/src/sgml/monitoring.sgml
@@ -1759,6 +1759,10 @@ postgres 27093 0.0 0.0 30096 2752 ? Ss 11:34 0:00 postgres: ser
<entry><literal>ParallelFinish</literal></entry>
<entry>Waiting for parallel workers to finish computing.</entry>
</row>
+ <row>
+ <entry><literal>ParallelVacuumFinish</literal></entry>
+ <entry>Waiting for parallel vacuum workers to finish computing.</entry>
+ </row>
<row>
<entry><literal>ProcArrayGroupUpdate</literal></entry>
<entry>Waiting for the group leader to clear the transaction ID at
@@ -6815,6 +6819,28 @@ FROM pg_stat_get_backend_idset() AS backendid;
Number of dead tuples collected since the last index vacuum cycle.
</para></entry>
</row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>indexes_total</structfield> <type>bigint</type>
+ </para>
+ <para>
+ Number of indexes that wil be vacuumed. This value will be
+ <literal>0</literal> if there are no indexes to vacuum, <literal>INDEX_CLEANUP</literal>
+ is set to <literal>OFF</literal>, or vacuum failsafe is triggered.
+ See <xref linkend="guc-vacuum-failsafe-age"/>
+ for more on vacuum failsafe.
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>indexes_completed</structfield> <type>bigint</type>
+ </para>
+ <para>
+ Number of indexes vacuumed in the current vacuum cycle.
+ </para></entry>
+ </row>
</tbody>
</tgroup>
</table>
diff --git a/src/backend/access/gin/ginvacuum.c b/src/backend/access/gin/ginvacuum.c
index b4fa5f6..e681164 100644
--- a/src/backend/access/gin/ginvacuum.c
+++ b/src/backend/access/gin/ginvacuum.c
@@ -17,12 +17,14 @@
#include "access/gin_private.h"
#include "access/ginxlog.h"
#include "access/xloginsert.h"
+#include "commands/progress.h"
#include "commands/vacuum.h"
#include "miscadmin.h"
#include "postmaster/autovacuum.h"
#include "storage/indexfsm.h"
#include "storage/lmgr.h"
#include "storage/predicate.h"
+#include "utils/backend_progress.h"
#include "utils/memutils.h"
struct GinVacuumState
@@ -665,6 +667,9 @@ ginbulkdelete(IndexVacuumInfo *info, IndexBulkDeleteResult *stats,
vacuum_delay_point();
+ if (info->report_parallel_progress && (blkno % REPORT_PARALLEL_VACUUM_EVERY_PAGES) == 0)
+ parallel_vacuum_update_progress();
+
for (i = 0; i < nRoot; i++)
{
ginVacuumPostingTree(&gvs, rootOfPostingTree[i]);
@@ -751,6 +756,9 @@ ginvacuumcleanup(IndexVacuumInfo *info, IndexBulkDeleteResult *stats)
vacuum_delay_point();
+ if (info->report_parallel_progress && (blkno % REPORT_PARALLEL_VACUUM_EVERY_PAGES) == 0)
+ parallel_vacuum_update_progress();
+
buffer = ReadBufferExtended(index, MAIN_FORKNUM, blkno,
RBM_NORMAL, info->strategy);
LockBuffer(buffer, GIN_SHARE);
diff --git a/src/backend/access/gist/gistvacuum.c b/src/backend/access/gist/gistvacuum.c
index 0aa6e58..cb3deb1 100644
--- a/src/backend/access/gist/gistvacuum.c
+++ b/src/backend/access/gist/gistvacuum.c
@@ -17,11 +17,13 @@
#include "access/genam.h"
#include "access/gist_private.h"
#include "access/transam.h"
+#include "commands/progress.h"
#include "commands/vacuum.h"
#include "lib/integerset.h"
#include "miscadmin.h"
#include "storage/indexfsm.h"
#include "storage/lmgr.h"
+#include "utils/backend_progress.h"
#include "utils/memutils.h"
/* Working state needed by gistbulkdelete */
@@ -223,7 +225,11 @@ gistvacuumscan(IndexVacuumInfo *info, IndexBulkDeleteResult *stats,
break;
/* Iterate over pages, then loop back to recheck length */
for (; blkno < num_pages; blkno++)
+ {
gistvacuumpage(&vstate, blkno, blkno);
+ if (info->report_parallel_progress && (blkno % REPORT_PARALLEL_VACUUM_EVERY_PAGES) == 0)
+ parallel_vacuum_update_progress();
+ }
}
/*
diff --git a/src/backend/access/hash/hash.c b/src/backend/access/hash/hash.c
index 77fd147..d807c9f 100644
--- a/src/backend/access/hash/hash.c
+++ b/src/backend/access/hash/hash.c
@@ -505,6 +505,12 @@ loop_top:
blkno = bucket_blkno;
+ /*
+ * For hash indexes, we report parallel vacuum progress
+ * for every bucket.
+ */
+ if (info->report_parallel_progress)
+ parallel_vacuum_update_progress();
/*
* We need to acquire a cleanup lock on the primary bucket page to out
* wait concurrent scans before deleting the dead tuples.
diff --git a/src/backend/access/heap/vacuumlazy.c b/src/backend/access/heap/vacuumlazy.c
index d59711b..f11a1b1 100644
--- a/src/backend/access/heap/vacuumlazy.c
+++ b/src/backend/access/heap/vacuumlazy.c
@@ -415,6 +415,7 @@ heap_vacuum_rel(Relation rel, VacuumParams *params,
vacrel->rel = rel;
vac_open_indexes(vacrel->rel, RowExclusiveLock, &vacrel->nindexes,
&vacrel->indrels);
+
if (instrument && vacrel->nindexes > 0)
{
/* Copy index names used by instrumentation (not error reporting) */
@@ -459,6 +460,10 @@ heap_vacuum_rel(Relation rel, VacuumParams *params,
Assert(params->index_cleanup == VACOPTVALUE_AUTO);
}
+ /* report number of indexes to vacuum, if we are told to cleanup indexes */
+ if (vacrel->do_index_cleanup)
+ pgstat_progress_update_param(PROGRESS_VACUUM_INDEX_TOTAL, vacrel->nindexes);
+
vacrel->bstrategy = bstrategy;
vacrel->relfrozenxid = rel->rd_rel->relfrozenxid;
vacrel->relminmxid = rel->rd_rel->relminmxid;
@@ -2301,6 +2306,12 @@ lazy_vacuum_all_indexes(LVRelState *vacrel)
lazy_vacuum_one_index(indrel, istat, vacrel->old_live_tuples,
vacrel);
+ /*
+ * Done vacuuming an index. Increment the indexes completed
+ */
+ pgstat_progress_update_param(PROGRESS_VACUUM_INDEX_COMPLETED,
+ idx + 1);
+
if (lazy_check_wraparound_failsafe(vacrel))
{
/* Wraparound emergency -- end current index scan */
@@ -2335,11 +2346,14 @@ lazy_vacuum_all_indexes(LVRelState *vacrel)
Assert(allindexes || vacrel->failsafe_active);
/*
- * Increase and report the number of index scans.
+ * Reset and report the number of indexes scanned.
+ * Also, increase and report the number of index
+ * scans.
*
* We deliberately include the case where we started a round of bulk
* deletes that we weren't able to finish due to the failsafe triggering.
*/
+ pgstat_progress_update_param(PROGRESS_VACUUM_INDEX_COMPLETED, 0);
vacrel->num_index_scans++;
pgstat_progress_update_param(PROGRESS_VACUUM_NUM_INDEX_VACUUMS,
vacrel->num_index_scans);
@@ -2593,10 +2607,17 @@ lazy_check_wraparound_failsafe(LVRelState *vacrel)
{
vacrel->failsafe_active = true;
- /* Disable index vacuuming, index cleanup, and heap rel truncation */
+ /*
+ * Disable index vacuuming, index cleanup, and heap rel truncation
+ *
+ * Also, report to progress.h that we are no longer tracking
+ * index vacuum/cleanup.
+ */
vacrel->do_index_vacuuming = false;
vacrel->do_index_cleanup = false;
vacrel->do_rel_truncate = false;
+ pgstat_progress_update_param(PROGRESS_VACUUM_INDEX_TOTAL, 0);
+ pgstat_progress_update_param(PROGRESS_VACUUM_INDEX_COMPLETED, 0);
ereport(WARNING,
(errmsg("bypassing nonessential maintenance of table \"%s.%s.%s\" as a failsafe after %d index scans",
@@ -2644,6 +2665,12 @@ lazy_cleanup_all_indexes(LVRelState *vacrel)
vacrel->indstats[idx] =
lazy_cleanup_one_index(indrel, istat, reltuples,
estimated_count, vacrel);
+
+ /*
+ * Done cleaning an index. Increment the indexes completed
+ */
+ pgstat_progress_update_param(PROGRESS_VACUUM_INDEX_COMPLETED,
+ idx + 1);
}
}
else
@@ -2678,6 +2705,7 @@ lazy_vacuum_one_index(Relation indrel, IndexBulkDeleteResult *istat,
ivinfo.index = indrel;
ivinfo.analyze_only = false;
ivinfo.report_progress = false;
+ ivinfo.report_parallel_progress = false;
ivinfo.estimated_count = true;
ivinfo.message_level = DEBUG2;
ivinfo.num_heap_tuples = reltuples;
@@ -2726,6 +2754,7 @@ lazy_cleanup_one_index(Relation indrel, IndexBulkDeleteResult *istat,
ivinfo.index = indrel;
ivinfo.analyze_only = false;
ivinfo.report_progress = false;
+ ivinfo.report_parallel_progress = false;
ivinfo.estimated_count = estimated_count;
ivinfo.message_level = DEBUG2;
diff --git a/src/backend/access/nbtree/nbtree.c b/src/backend/access/nbtree/nbtree.c
index b52eca8..39349bb 100644
--- a/src/backend/access/nbtree/nbtree.c
+++ b/src/backend/access/nbtree/nbtree.c
@@ -998,6 +998,8 @@ btvacuumscan(IndexVacuumInfo *info, IndexBulkDeleteResult *stats,
if (info->report_progress)
pgstat_progress_update_param(PROGRESS_SCAN_BLOCKS_DONE,
scanblkno);
+ if (info->report_parallel_progress && (scanblkno % REPORT_PARALLEL_VACUUM_EVERY_PAGES) == 0)
+ parallel_vacuum_update_progress();
}
}
diff --git a/src/backend/access/spgist/spgvacuum.c b/src/backend/access/spgist/spgvacuum.c
index ad90b21..0589683 100644
--- a/src/backend/access/spgist/spgvacuum.c
+++ b/src/backend/access/spgist/spgvacuum.c
@@ -21,11 +21,13 @@
#include "access/transam.h"
#include "access/xloginsert.h"
#include "catalog/storage_xlog.h"
+#include "commands/progress.h"
#include "commands/vacuum.h"
#include "miscadmin.h"
#include "storage/bufmgr.h"
#include "storage/indexfsm.h"
#include "storage/lmgr.h"
+#include "utils/backend_progress.h"
#include "utils/snapmgr.h"
@@ -843,6 +845,9 @@ spgvacuumscan(spgBulkDeleteState *bds)
/* empty the pending-list after each page */
if (bds->pendingList != NULL)
spgprocesspending(bds);
+ /* report parallel vacuum progress */
+ if (bds->info->report_parallel_progress && (blkno % REPORT_PARALLEL_VACUUM_EVERY_PAGES) == 0)
+ parallel_vacuum_update_progress();
}
}
diff --git a/src/backend/catalog/index.c b/src/backend/catalog/index.c
index 61f1d39..11b3212 100644
--- a/src/backend/catalog/index.c
+++ b/src/backend/catalog/index.c
@@ -3348,6 +3348,7 @@ validate_index(Oid heapId, Oid indexId, Snapshot snapshot)
ivinfo.index = indexRelation;
ivinfo.analyze_only = false;
ivinfo.report_progress = true;
+ ivinfo.report_parallel_progress = false;
ivinfo.estimated_count = true;
ivinfo.message_level = DEBUG2;
ivinfo.num_heap_tuples = heapRelation->rd_rel->reltuples;
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index 2d8104b..c37b20b 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1165,7 +1165,8 @@ CREATE VIEW pg_stat_progress_vacuum AS
END AS phase,
S.param2 AS heap_blks_total, S.param3 AS heap_blks_scanned,
S.param4 AS heap_blks_vacuumed, S.param5 AS index_vacuum_count,
- S.param6 AS max_dead_tuples, S.param7 AS num_dead_tuples
+ S.param6 AS max_dead_tuples, S.param7 AS num_dead_tuples,
+ S.param8 AS indexes_total, S.param9 AS indexes_completed
FROM pg_stat_get_progress_info('VACUUM') AS S
LEFT JOIN pg_database D ON S.datid = D.oid;
diff --git a/src/backend/commands/vacuum.c b/src/backend/commands/vacuum.c
index a6d5ed1..2da956f 100644
--- a/src/backend/commands/vacuum.c
+++ b/src/backend/commands/vacuum.c
@@ -75,6 +75,8 @@ int vacuum_multixact_failsafe_age;
static MemoryContext vac_context = NULL;
static BufferAccessStrategy vac_strategy;
+/* Shared parameter to track vacuum parallel progress */
+pg_atomic_uint32 *ParallelVacuumProgress = NULL;
/*
* Variables for cost-based parallel vacuum. See comments atop
diff --git a/src/backend/commands/vacuumparallel.c b/src/backend/commands/vacuumparallel.c
index f26d796..b5b80c9 100644
--- a/src/backend/commands/vacuumparallel.c
+++ b/src/backend/commands/vacuumparallel.c
@@ -30,6 +30,7 @@
#include "access/table.h"
#include "access/xact.h"
#include "catalog/index.h"
+#include "commands/progress.h"
#include "commands/vacuum.h"
#include "optimizer/paths.h"
#include "pgstat.h"
@@ -50,6 +51,8 @@
#define PARALLEL_VACUUM_KEY_WAL_USAGE 5
#define PARALLEL_VACUUM_KEY_INDEX_STATS 6
+#define PARALLEL_VACUUM_PROGRESS_TIMEOUT 1000
+
/*
* Shared information among parallel workers. So this is allocated in the DSM
* segment.
@@ -103,6 +106,17 @@ typedef struct PVShared
/* Counter for vacuuming and cleanup */
pg_atomic_uint32 idx;
+
+ /*
+ * Counter for vacuuming and cleanup progress reporting.
+ * This value is used to report index vacuum/cleanup progress
+ * in parallel_vacuum_progress_report. We keep this
+ * counter to avoid having to loop through
+ * ParallelVacuumState->indstats to determine the number
+ * of indexes completed.
+ */
+ pg_atomic_uint32 idx_completed_progress;
+
} PVShared;
/* Status used during parallel index vacuum or cleanup */
@@ -213,6 +227,7 @@ static void parallel_vacuum_process_one_index(ParallelVacuumState *pvs, Relation
static bool parallel_vacuum_index_is_parallel_safe(Relation indrel, int num_index_scans,
bool vacuum);
static void parallel_vacuum_error_callback(void *arg);
+static void parallel_wait_for_workers_to_finish(ParallelVacuumState *pvs);
/*
* Try to enter parallel mode and create a parallel context. Then initialize
@@ -364,6 +379,7 @@ parallel_vacuum_init(Relation rel, Relation *indrels, int nindexes,
pg_atomic_init_u32(&(shared->cost_balance), 0);
pg_atomic_init_u32(&(shared->active_nworkers), 0);
pg_atomic_init_u32(&(shared->idx), 0);
+ pg_atomic_init_u32(&(shared->idx_completed_progress), 0);
shm_toc_insert(pcxt->toc, PARALLEL_VACUUM_KEY_SHARED, shared);
pvs->shared = shared;
@@ -618,8 +634,9 @@ parallel_vacuum_process_all_indexes(ParallelVacuumState *pvs, int num_index_scan
vacuum));
}
- /* Reset the parallel index processing counter */
+ /* Reset the parallel index processing counter ( index progress counter also ) */
pg_atomic_write_u32(&(pvs->shared->idx), 0);
+ pg_atomic_write_u32(&(pvs->shared->idx_completed_progress), 0);
/* Setup the shared cost-based vacuum delay and launch workers */
if (nworkers > 0)
@@ -645,6 +662,14 @@ parallel_vacuum_process_all_indexes(ParallelVacuumState *pvs, int num_index_scan
LaunchParallelWorkers(pvs->pcxt);
+ /*
+ * Set the shared parallel vacuum progress. This will be used
+ * to periodically update progress.h with completed indexes
+ * in a parallel vacuum. See comments in parallel_vacuum_update_progress
+ * for more details.
+ */
+ ParallelVacuumProgress = &(pvs->shared->idx_completed_progress);
+
if (pvs->pcxt->nworkers_launched > 0)
{
/*
@@ -688,7 +713,21 @@ parallel_vacuum_process_all_indexes(ParallelVacuumState *pvs, int num_index_scan
*/
if (nworkers > 0)
{
- /* Wait for all vacuum workers to finish */
+ /*
+ * To wait for parallel workers to finish,
+ * first call parallel_wait_for_workers_to_finish
+ * which is responsible for reporting the
+ * number of indexes completed.
+ *
+ * Afterwards, WaitForParallelWorkersToFinish is called
+ * to do the real work of waiting for parallel workers
+ * to finish.
+ *
+ * Note: Both routines will acquire a WaitLatch in their
+ * respective loops.
+ */
+ parallel_wait_for_workers_to_finish(pvs);
+
WaitForParallelWorkersToFinish(pvs->pcxt);
for (int i = 0; i < pvs->pcxt->nworkers_launched; i++)
@@ -710,6 +749,9 @@ parallel_vacuum_process_all_indexes(ParallelVacuumState *pvs, int num_index_scan
indstats->status = PARALLEL_INDVAC_STATUS_INITIAL;
}
+ /* Reset parallel progress */
+ ParallelVacuumProgress = NULL;
+
/*
* Carry the shared balance value to heap scan and disable shared costing
*/
@@ -838,7 +880,8 @@ parallel_vacuum_process_one_index(ParallelVacuumState *pvs, Relation indrel,
ivinfo.estimated_count = pvs->shared->estimated_count;
ivinfo.num_heap_tuples = pvs->shared->reltuples;
ivinfo.strategy = pvs->bstrategy;
-
+ /* Only the leader should report parallel vacuum progress */
+ ivinfo.report_parallel_progress = !IsParallelWorker();
/* Update error traceback information */
pvs->indname = pstrdup(RelationGetRelationName(indrel));
pvs->status = indstats->status;
@@ -857,6 +900,9 @@ parallel_vacuum_process_one_index(ParallelVacuumState *pvs, Relation indrel,
RelationGetRelationName(indrel));
}
+ if (ivinfo.report_parallel_progress)
+ parallel_vacuum_update_progress();
+
/*
* Copy the index bulk-deletion result returned from ambulkdelete and
* amvacuumcleanup to the DSM segment if it's the first cycle because they
@@ -888,6 +934,9 @@ parallel_vacuum_process_one_index(ParallelVacuumState *pvs, Relation indrel,
pvs->status = PARALLEL_INDVAC_STATUS_COMPLETED;
pfree(pvs->indname);
pvs->indname = NULL;
+
+ /* Update the number of indexes completed. */
+ pg_atomic_add_fetch_u32(&(pvs->shared->idx_completed_progress), 1);
}
/*
@@ -1072,3 +1121,58 @@ parallel_vacuum_error_callback(void *arg)
return;
}
}
+
+/*
+ * Read the shared ParallelVacuumProgress and update progress.h
+ * with indexes vacuumed so far. This function is called periodically
+ * by index AMs as well as parallel_vacuum_process_one_index.
+ *
+ * To avoid unnecessarily updating progress, we check the progress
+ * values from the backend entry and only update if the value
+ * of completed indexes increases.
+ *
+ * Note: This function should be used by the leader process only,
+ * and it's up to the caller to ensure this.
+ */
+void
+parallel_vacuum_update_progress(void)
+{
+ volatile PgBackendStatus *beentry = MyBEEntry;
+
+ Assert(!IsParallelWorker);
+
+ if (beentry && ParallelVacuumProgress)
+ {
+ int parallel_vacuum_current_value = beentry->st_progress_param[PROGRESS_VACUUM_INDEX_COMPLETED];
+ int parallel_vacuum_new_value = pg_atomic_read_u32(ParallelVacuumProgress);
+
+ if (parallel_vacuum_new_value > parallel_vacuum_current_value)
+ pgstat_progress_update_param(PROGRESS_VACUUM_INDEX_COMPLETED, parallel_vacuum_new_value);
+ }
+}
+
+/*
+ * Check if we are done vacuuming indexes and report
+ * progress.
+ *
+ * We nap using with a WaitLatch to avoid a busy loop.
+ *
+ * Note: This function should be used by the leader process only,
+ * and it's up to the caller to ensure this.
+ */
+void
+parallel_wait_for_workers_to_finish(ParallelVacuumState *pvs)
+{
+ Assert(!IsParallelWorker);
+
+ while (pg_atomic_read_u32(ParallelVacuumProgress) < pvs->nindexes)
+ {
+ CHECK_FOR_INTERRUPTS();
+
+ parallel_vacuum_update_progress();
+
+ (void) WaitLatch(MyLatch, WL_TIMEOUT | WL_LATCH_SET | WL_EXIT_ON_PM_DEATH, PARALLEL_VACUUM_PROGRESS_TIMEOUT,
+ WAIT_EVENT_PARALLEL_VACUUM_FINISH);
+ ResetLatch(MyLatch);
+ }
+}
diff --git a/src/backend/utils/activity/wait_event.c b/src/backend/utils/activity/wait_event.c
index b2abd75..eeba2be 100644
--- a/src/backend/utils/activity/wait_event.c
+++ b/src/backend/utils/activity/wait_event.c
@@ -460,6 +460,9 @@ pgstat_get_wait_ipc(WaitEventIPC w)
case WAIT_EVENT_XACT_GROUP_UPDATE:
event_name = "XactGroupUpdate";
break;
+ case WAIT_EVENT_PARALLEL_VACUUM_FINISH:
+ event_name = "ParallelVacuumFinish";
+ break;
/* no default case, so that compiler will warn */
}
diff --git a/src/include/access/genam.h b/src/include/access/genam.h
index e1c4fdb..7474734 100644
--- a/src/include/access/genam.h
+++ b/src/include/access/genam.h
@@ -46,6 +46,7 @@ typedef struct IndexVacuumInfo
Relation index; /* the index being vacuumed */
bool analyze_only; /* ANALYZE (without any actual vacuum) */
bool report_progress; /* emit progress.h status reports */
+ bool report_parallel_progress; /* emit progress.h status reports for parallel vacuum */
bool estimated_count; /* num_heap_tuples is an estimate */
int message_level; /* ereport level for progress messages */
double num_heap_tuples; /* tuples remaining in heap */
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index a28938c..0e97c6d 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -25,6 +25,8 @@
#define PROGRESS_VACUUM_NUM_INDEX_VACUUMS 4
#define PROGRESS_VACUUM_MAX_DEAD_TUPLES 5
#define PROGRESS_VACUUM_NUM_DEAD_TUPLES 6
+#define PROGRESS_VACUUM_INDEX_TOTAL 7
+#define PROGRESS_VACUUM_INDEX_COMPLETED 8
/* Phases of vacuum (as advertised via PROGRESS_VACUUM_PHASE) */
#define PROGRESS_VACUUM_PHASE_SCAN_HEAP 1
diff --git a/src/include/commands/vacuum.h b/src/include/commands/vacuum.h
index 4e4bc26..5a6b454 100644
--- a/src/include/commands/vacuum.h
+++ b/src/include/commands/vacuum.h
@@ -64,6 +64,12 @@
/* value for checking vacuum flags */
#define VACUUM_OPTION_MAX_VALID_VALUE ((1 << 3) - 1)
+/*
+ * Parallel Index vacuum progress is reported every 1GB of blocks
+ * scanned.
+ */
+#define REPORT_PARALLEL_VACUUM_EVERY_PAGES ((BlockNumber) (((uint64) 1024 * 1024 * 1024) / BLCKSZ))
+
/* Abstract type for parallel vacuum state */
typedef struct ParallelVacuumState ParallelVacuumState;
@@ -259,6 +265,8 @@ extern PGDLLIMPORT int vacuum_multixact_freeze_table_age;
extern PGDLLIMPORT int vacuum_failsafe_age;
extern PGDLLIMPORT int vacuum_multixact_failsafe_age;
+extern PGDLLIMPORT pg_atomic_uint32 *ParallelVacuumProgress;
+
/* Variables for cost-based parallel vacuum */
extern PGDLLIMPORT pg_atomic_uint32 *VacuumSharedCostBalance;
extern PGDLLIMPORT pg_atomic_uint32 *VacuumActiveNWorkers;
@@ -333,5 +341,6 @@ extern bool std_typanalyze(VacAttrStats *stats);
extern double anl_random_fract(void);
extern double anl_init_selection_state(int n);
extern double anl_get_next_S(double t, int n, double *stateptr);
+extern void parallel_vacuum_update_progress(void);
#endif /* VACUUM_H */
diff --git a/src/include/utils/wait_event.h b/src/include/utils/wait_event.h
index 0b2100b..95e9fef 100644
--- a/src/include/utils/wait_event.h
+++ b/src/include/utils/wait_event.h
@@ -128,7 +128,8 @@ typedef enum
WAIT_EVENT_SYNC_REP,
WAIT_EVENT_WAL_RECEIVER_EXIT,
WAIT_EVENT_WAL_RECEIVER_WAIT_START,
- WAIT_EVENT_XACT_GROUP_UPDATE
+ WAIT_EVENT_XACT_GROUP_UPDATE,
+ WAIT_EVENT_PARALLEL_VACUUM_FINISH
} WaitEventIPC;
/* ----------
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index 37c1c86..896043e 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -2021,7 +2021,9 @@ pg_stat_progress_vacuum| SELECT s.pid,
s.param4 AS heap_blks_vacuumed,
s.param5 AS index_vacuum_count,
s.param6 AS max_dead_tuples,
- s.param7 AS num_dead_tuples
+ s.param7 AS num_dead_tuples,
+ s.param8 AS indexes_total,
+ s.param9 AS indexes_completed
FROM (pg_stat_get_progress_info('VACUUM'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
LEFT JOIN pg_database d ON ((s.datid = d.oid)));
pg_stat_recovery_prefetch| SELECT s.stats_reset,
--
2.32.1 (Apple Git-133)
^ permalink raw reply [nested|flat] 44+ messages in thread
* Re: Add index scan progress to pg_stat_progress_vacuum
@ 2022-12-06 03:18 Masahiko Sawada <[email protected]>
parent: Imseih (AWS), Sami <[email protected]>
0 siblings, 1 reply; 44+ messages in thread
From: Masahiko Sawada @ 2022-12-06 03:18 UTC (permalink / raw)
To: Imseih (AWS), Sami <[email protected]>; +Cc: Andres Freund <[email protected]>; Robert Haas <[email protected]>; Bossart, Nathan <[email protected]>; Peter Geoghegan <[email protected]>; Justin Pryzby <[email protected]>; pgsql-hackers
Hi,
Thank you for updating the patch!
On Tue, Nov 29, 2022 at 8:57 AM Imseih (AWS), Sami <[email protected]> wrote:
>
> > I think that indexes_total should be 0 also when INDEX_CLEANUP is off.
>
> Patch updated for handling of INDEX_CLEANUP = off, with an update to
> the documentation as well.
>
> > I think we don't need to reset it at the end of index vacuuming. There
> > is a small window before switching to the next phase. If we reset this
> > value while showing "index vacuuming" phase, the user might get
> > confused. Instead, we can reset it at the beginning of the index
> > vacuuming.
>
> No, I think the way it's currently done is correct. We want to reset the number
> of indexes completed before we increase the num_index_scans ( index vacuum cycle ).
> This ensures that when we enter a new index cycle, the number of indexes completed
> are already reset. The 2 fields that matter here is how many indexes are vacuumed in the
> currently cycle and this is called out in the documentation as such.
>
Agreed.
Here are comments on v16 patch.
--- a/contrib/bloom/blvacuum.c
+++ b/contrib/bloom/blvacuum.c
@@ -15,12 +15,14 @@
#include "access/genam.h"
#include "bloom.h"
#include "catalog/storage.h"
+#include "commands/progress.h"
#include "commands/vacuum.h"
#include "miscadmin.h"
#include "postmaster/autovacuum.h"
#include "storage/bufmgr.h"
#include "storage/indexfsm.h"
#include "storage/lmgr.h"
+#include "utils/backend_progress.h"
I think we don't need to include them here. Probably the same is true
for other index AMs.
---
vacuum_delay_point();
+ if (info->report_parallel_progress && (blkno %
REPORT_PARALLEL_VACUUM_EVERY_PAGES) == 0)
+ parallel_vacuum_update_progress();
+
buffer = ReadBufferExtended(index, MAIN_FORKNUM, blkno,
There is an extra new line.
---
+ <row>
+ <entry><literal>ParallelVacuumFinish</literal></entry>
+ <entry>Waiting for parallel vacuum workers to finish computing.</entry>
+ </row>
How about "Waiting for parallel vacuum workers to finish index vacuum"?
---
vacrel->rel = rel;
vac_open_indexes(vacrel->rel, RowExclusiveLock, &vacrel->nindexes,
&vacrel->indrels);
+
if (instrument && vacrel->nindexes > 0)
{
/* Copy index names used by instrumentation (not error reporting) */
This added line is not necessary.
---
/* Counter for vacuuming and cleanup */
pg_atomic_uint32 idx;
+
+ /*
+ * Counter for vacuuming and cleanup progress reporting.
+ * This value is used to report index vacuum/cleanup progress
+ * in parallel_vacuum_progress_report. We keep this
+ * counter to avoid having to loop through
+ * ParallelVacuumState->indstats to determine the number
+ * of indexes completed.
+ */
+ pg_atomic_uint32 idx_completed_progress;
I think the name of idx_completed_progress is very confusing. Since
the idx of PVShared refers to the current index in the pvs->indstats[]
whereas idx_completed_progress is the number of vacuumed indexes. How
about "nindexes_completed"?
---
+ /*
+ * Set the shared parallel vacuum progress. This will be used
+ * to periodically update progress.h with completed indexes
+ * in a parallel vacuum. See comments in
parallel_vacuum_update_progress
+ * for more details.
+ */
+ ParallelVacuumProgress =
&(pvs->shared->idx_completed_progress);
+
Since we pass pvs to parallel_wait_for_workers_to_finish(), we don't
need to have ParallelVacuumProgress. I see
parallel_vacuum_update_progress() uses this value but I think it's
better to pass ParallelVacuumState to via IndexVacuumInfo.
---
+ /*
+ * To wait for parallel workers to finish,
+ * first call parallel_wait_for_workers_to_finish
+ * which is responsible for reporting the
+ * number of indexes completed.
+ *
+ * Afterwards, WaitForParallelWorkersToFinish is called
+ * to do the real work of waiting for parallel workers
+ * to finish.
+ *
+ * Note: Both routines will acquire a WaitLatch in their
+ * respective loops.
+ */
How about something like:
Wait for all indexes to be vacuumed while updating the parallel vacuum
index progress. And then wait for all workers to finish.
---
RelationGetRelationName(indrel));
}
+ if (ivinfo.report_parallel_progress)
+ parallel_vacuum_update_progress();
+
I think it's better to update the progress info after updating
pvs->shared->idx_completed_progress.
---
+/*
+ * Check if we are done vacuuming indexes and report
+ * progress.
How about "Waiting for all indexes to be vacuumed while updating the
parallel index vacuum progress"?
+ *
+ * We nap using with a WaitLatch to avoid a busy loop.
+ *
+ * Note: This function should be used by the leader process only,
+ * and it's up to the caller to ensure this.
+ */
I think these comments are not necessary.
+void
+parallel_wait_for_workers_to_finish(ParallelVacuumState *pvs)
How about "parallel_vacuum_wait_to_finish"?
---
+/*
+ * Read the shared ParallelVacuumProgress and update progress.h
+ * with indexes vacuumed so far. This function is called periodically
+ * by index AMs as well as parallel_vacuum_process_one_index.
+ *
+ * To avoid unnecessarily updating progress, we check the progress
+ * values from the backend entry and only update if the value
+ * of completed indexes increases.
+ *
+ * Note: This function should be used by the leader process only,
+ * and it's up to the caller to ensure this.
+ */
+void
+parallel_vacuum_update_progress(void)
+{
+ volatile PgBackendStatus *beentry = MyBEEntry;
+
+ Assert(!IsParallelWorker);
+
+ if (beentry && ParallelVacuumProgress)
+ {
+ int parallel_vacuum_current_value =
beentry->st_progress_param[PROGRESS_VACUUM_INDEX_COMPLETED];
+ int parallel_vacuum_new_value =
pg_atomic_read_u32(ParallelVacuumProgress);
+
+ if (parallel_vacuum_new_value > parallel_vacuum_current_value)
+
pgstat_progress_update_param(PROGRESS_VACUUM_INDEX_COMPLETED,
parallel_vacuum_new_value);
+ }
+}
parallel_vacuum_update_progress() is typically called every 1GB so I
think we don't need to worry about unnecessary update. Also, I think
this code doesn't work when pgstat_track_activities is false. Instead,
I think that in parallel_wait_for_workers_to_finish(), we can check
the value of pvs->nindexes_completed and update the progress if there
is an update or it's first time.
---
+ (void) WaitLatch(MyLatch, WL_TIMEOUT | WL_LATCH_SET |
WL_EXIT_ON_PM_DEATH, PARALLEL_VACUUM_PROGRESS_TIMEOUT,
+
WAIT_EVENT_PARALLEL_VACUUM_FINISH);
+ ResetLatch(MyLatch);
I think we don't necessarily need to use
PARALLEL_VACUUM_PROGRESS_TIMEOUT here. Probably we can use 1000L
instead. If we want to use PARALLEL_VACUUM_PROGRESS_TIMEOUT, we need
comments for that:
+#define PARALLEL_VACUUM_PROGRESS_TIMEOUT 1000
---
- WAIT_EVENT_XACT_GROUP_UPDATE
+ WAIT_EVENT_XACT_GROUP_UPDATE,
+ WAIT_EVENT_PARALLEL_VACUUM_FINISH
} WaitEventIPC;
Enums of WaitEventIPC should be defined in alphabetical order.
---
cfbot fails.
Regards,
--
Masahiko Sawada
Amazon Web Services: https://aws.amazon.com
^ permalink raw reply [nested|flat] 44+ messages in thread
* Re: Add index scan progress to pg_stat_progress_vacuum
@ 2022-12-13 04:40 Imseih (AWS), Sami <[email protected]>
parent: Masahiko Sawada <[email protected]>
0 siblings, 1 reply; 44+ messages in thread
From: Imseih (AWS), Sami @ 2022-12-13 04:40 UTC (permalink / raw)
To: Masahiko Sawada <[email protected]>; +Cc: Andres Freund <[email protected]>; Robert Haas <[email protected]>; Bossart, Nathan <[email protected]>; Peter Geoghegan <[email protected]>; Justin Pryzby <[email protected]>; pgsql-hackers
Thanks for the feedback. I agree with the feedback, except
for
> need to have ParallelVacuumProgress. I see
> parallel_vacuum_update_progress() uses this value but I think it's
> better to pass ParallelVacuumState to via IndexVacuumInfo.
I was trying to avoid passing a pointer to
ParallelVacuumState in IndexVacuuminfo.
ParallelVacuumProgress is implemented in the same
way as VacuumSharedCostBalance and
VacuumActiveNWorkers. See vacuum.h
These values are reset at the start of a parallel vacuum cycle
and reset at the end of an index vacuum cycle.
This seems like a better approach and less invasive.
What would be a reason not to go with this approach?
> parallel_vacuum_update_progress() is typically called every 1GB so I
> think we don't need to worry about unnecessary update. Also, I think
> this code doesn't work when pgstat_track_activities is false. Instead,
> I think that in parallel_wait_for_workers_to_finish(), we can check
> the value of pvs->nindexes_completed and update the progress if there
> is an update or it's first time.
I agree that we don’t need to worry about unnecessary updates
in parallel_vacuum_update_progress since we are calling
every 1GB. I also don't think we should do anything additional
in parallel_wait_for_workers_to_finish since here we are only
updating every 1 second.
Thanks,
Sami Imseih
Amazon Web Services
^ permalink raw reply [nested|flat] 44+ messages in thread
* Re: Add index scan progress to pg_stat_progress_vacuum
@ 2022-12-14 01:43 Masahiko Sawada <[email protected]>
parent: Imseih (AWS), Sami <[email protected]>
0 siblings, 1 reply; 44+ messages in thread
From: Masahiko Sawada @ 2022-12-14 01:43 UTC (permalink / raw)
To: Imseih (AWS), Sami <[email protected]>; +Cc: Andres Freund <[email protected]>; Robert Haas <[email protected]>; Bossart, Nathan <[email protected]>; Peter Geoghegan <[email protected]>; Justin Pryzby <[email protected]>; pgsql-hackers
On Tue, Dec 13, 2022 at 1:40 PM Imseih (AWS), Sami <[email protected]> wrote:
>
> Thanks for the feedback. I agree with the feedback, except
> for
>
> > need to have ParallelVacuumProgress. I see
> > parallel_vacuum_update_progress() uses this value but I think it's
> > better to pass ParallelVacuumState to via IndexVacuumInfo.
>
> I was trying to avoid passing a pointer to
> ParallelVacuumState in IndexVacuuminfo.
>
> ParallelVacuumProgress is implemented in the same
> way as VacuumSharedCostBalance and
> VacuumActiveNWorkers. See vacuum.h
>
> These values are reset at the start of a parallel vacuum cycle
> and reset at the end of an index vacuum cycle.
>
> This seems like a better approach and less invasive.
> What would be a reason not to go with this approach?
First of all, I don't think we need to declare ParallelVacuumProgress
in vacuum.c since it's set and used only in vacuumparallel.c. But I
don't even think it's a good idea to declare it in vacuumparallel.c as
a static variable. The primary reason is that it adds things we need
to care about. For example, what if we raise an error during index
vacuum? The transaction aborts but ParallelVacuumProgress still refers
to something old. Suppose further that the next parallel vacuum
doesn't launch any workers, the leader process would still end up
accessing the old value pointed by ParallelVacuumProgress, which
causes a SEGV. So we need to reset it anyway at the beginning of the
parallel vacuum. It's easy to fix at this time but once the parallel
vacuum code gets more complex, it could forget to care about it.
IMO VacuumSharedCostBalance and VacuumActiveNWorkers have a different
story. They are set in vacuumparallel.c and are used in vacuum.c for
vacuum delay. If they weren't global variables, we would need to pass
them to every function that could eventually call the vacuum delay
function. So it makes sense to me to have them as global variables.On
the other hand, for ParallelVacuumProgress, it's a common pattern that
ambulkdelete(), amvacuumcleanup() or a common index scan routine like
btvacuumscan() checks the progress. I don't think index AM needs to
pass the value down to many of its functions. So it makes sense to me
to pass it via IndexVacuumInfo.
Having said that, I'd like to hear opinions also from other hackers, I
might be wrong and it's more invasive as you pointed out.
Regards,
--
Masahiko Sawada
Amazon Web Services: https://aws.amazon.com
^ permalink raw reply [nested|flat] 44+ messages in thread
* Re: Add index scan progress to pg_stat_progress_vacuum
@ 2022-12-14 05:09 Imseih (AWS), Sami <[email protected]>
parent: Masahiko Sawada <[email protected]>
0 siblings, 1 reply; 44+ messages in thread
From: Imseih (AWS), Sami @ 2022-12-14 05:09 UTC (permalink / raw)
To: Masahiko Sawada <[email protected]>; +Cc: Andres Freund <[email protected]>; Robert Haas <[email protected]>; Bossart, Nathan <[email protected]>; Peter Geoghegan <[email protected]>; Justin Pryzby <[email protected]>; pgsql-hackers
> First of all, I don't think we need to declare ParallelVacuumProgress
> in vacuum.c since it's set and used only in vacuumparallel.c. But I
> don't even think it's a good idea to declare it in vacuumparallel.c as
> a static variable. The primary reason is that it adds things we need
> to care about. For example, what if we raise an error during index
> vacuum? The transaction aborts but ParallelVacuumProgress still refers
> to something old. Suppose further that the next parallel vacuum
> doesn't launch any workers, the leader process would still end up
> accessing the old value pointed by ParallelVacuumProgress, which
> causes a SEGV. So we need to reset it anyway at the beginning of the
> parallel vacuum. It's easy to fix at this time but once the parallel
> vacuum code gets more complex, it could forget to care about it.
> IMO VacuumSharedCostBalance and VacuumActiveNWorkers have a different
> story. They are set in vacuumparallel.c and are used in vacuum.c for
> vacuum delay. If they weren't global variables, we would need to pass
> them to every function that could eventually call the vacuum delay
> function. So it makes sense to me to have them as global variables.On
> the other hand, for ParallelVacuumProgress, it's a common pattern that
> ambulkdelete(), amvacuumcleanup() or a common index scan routine like
> btvacuumscan() checks the progress. I don't think index AM needs to
> pass the value down to many of its functions. So it makes sense to me
> to pass it via IndexVacuumInfo.
Thanks for the detailed explanation and especially clearing up
my understanding of VacuumSharedCostBalance and VacuumActiveNWorker.
I do now think that passing ParallelVacuumState in IndexVacuumInfo is
a more optimal choice.
Attached version addresses the above and the previous comments.
Thanks
Sami Imseih
Amazon Web Services (AWS)
Attachments:
[application/octet-stream] v17-0001-Add-2-new-columns-to-pg_stat_progress_vacuum.-Th.patch (22.4K, ../../[email protected]/2-v17-0001-Add-2-new-columns-to-pg_stat_progress_vacuum.-Th.patch)
download | inline diff:
From 1fce3b7657c589cbf0fa049fd41470c7d713c492 Mon Sep 17 00:00:00 2001
From: "Imseih (AWS)" <[email protected]>
Date: Tue, 13 Dec 2022 23:02:24 -0600
Subject: [PATCH 1/1] Report index vacuum progress.
Add 2 new columns to pg_stat_progress_vacuum.
The columns are ndexes_total as the total indexes to be vacuumed or cleaned and
indexes_processed as the number of indexes vacuumed or cleaned up so far.
Author: Sami Imseih, based on suggestions by Nathan Bossart, Peter Geoghegan and Masahiko Sawada
Reviewed by: Nathan Bossart, Masahiko Sawada
Discussion: https://www.postgresql.org/message-id/flat/[email protected]
---
contrib/bloom/blvacuum.c | 4 ++
doc/src/sgml/monitoring.sgml | 26 +++++++++
src/backend/access/gin/ginvacuum.c | 6 ++
src/backend/access/gist/gistvacuum.c | 4 ++
src/backend/access/hash/hash.c | 6 ++
src/backend/access/heap/vacuumlazy.c | 34 +++++++++++-
src/backend/access/nbtree/nbtree.c | 2 +
src/backend/access/spgist/spgvacuum.c | 3 +
src/backend/catalog/index.c | 2 +
src/backend/catalog/system_views.sql | 3 +-
src/backend/commands/analyze.c | 1 +
src/backend/commands/vacuumparallel.c | 74 ++++++++++++++++++++++++-
src/backend/utils/activity/wait_event.c | 3 +
src/include/access/genam.h | 5 +-
src/include/commands/progress.h | 2 +
src/include/commands/vacuum.h | 7 +++
src/include/utils/wait_event.h | 1 +
src/test/regress/expected/rules.out | 4 +-
18 files changed, 180 insertions(+), 7 deletions(-)
diff --git a/contrib/bloom/blvacuum.c b/contrib/bloom/blvacuum.c
index 91fae5b0c0..005858ce9f 100644
--- a/contrib/bloom/blvacuum.c
+++ b/contrib/bloom/blvacuum.c
@@ -62,6 +62,8 @@ blbulkdelete(IndexVacuumInfo *info, IndexBulkDeleteResult *stats,
*itupEnd;
vacuum_delay_point();
+ if (info->report_parallel_progress && (blkno % REPORT_PARALLEL_VACUUM_EVERY_PAGES) == 0)
+ parallel_vacuum_update_progress(info->parallel_vacuum_state);
buffer = ReadBufferExtended(index, MAIN_FORKNUM, blkno,
RBM_NORMAL, info->strategy);
@@ -192,6 +194,8 @@ blvacuumcleanup(IndexVacuumInfo *info, IndexBulkDeleteResult *stats)
Page page;
vacuum_delay_point();
+ if (info->report_parallel_progress && (blkno % REPORT_PARALLEL_VACUUM_EVERY_PAGES) == 0)
+ parallel_vacuum_update_progress(info->parallel_vacuum_state);
buffer = ReadBufferExtended(index, MAIN_FORKNUM, blkno,
RBM_NORMAL, info->strategy);
diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml
index 11a8ebe5ec..22bc3bdd4d 100644
--- a/doc/src/sgml/monitoring.sgml
+++ b/doc/src/sgml/monitoring.sgml
@@ -1760,6 +1760,10 @@ postgres 27093 0.0 0.0 30096 2752 ? Ss 11:34 0:00 postgres: ser
<entry><literal>ParallelFinish</literal></entry>
<entry>Waiting for parallel workers to finish computing.</entry>
</row>
+ <row>
+ <entry><literal>ParallelVacuumFinish</literal></entry>
+ <entry>Waiting for parallel vacuum workers to finish index vacuum.</entry>
+ </row>
<row>
<entry><literal>ProcArrayGroupUpdate</literal></entry>
<entry>Waiting for the group leader to clear the transaction ID at
@@ -6817,6 +6821,28 @@ FROM pg_stat_get_backend_idset() AS backendid;
Number of dead tuples collected since the last index vacuum cycle.
</para></entry>
</row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>indexes_total</structfield> <type>bigint</type>
+ </para>
+ <para>
+ Number of indexes that wil be vacuumed. This value will be
+ <literal>0</literal> if there are no indexes to vacuum, <literal>INDEX_CLEANUP</literal>
+ is set to <literal>OFF</literal>, or vacuum failsafe is triggered.
+ See <xref linkend="guc-vacuum-failsafe-age"/>
+ for more on vacuum failsafe.
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>indexes_completed</structfield> <type>bigint</type>
+ </para>
+ <para>
+ Number of indexes vacuumed in the current vacuum cycle.
+ </para></entry>
+ </row>
</tbody>
</tgroup>
</table>
diff --git a/src/backend/access/gin/ginvacuum.c b/src/backend/access/gin/ginvacuum.c
index b4fa5f6bf8..0af63ebe48 100644
--- a/src/backend/access/gin/ginvacuum.c
+++ b/src/backend/access/gin/ginvacuum.c
@@ -665,6 +665,9 @@ ginbulkdelete(IndexVacuumInfo *info, IndexBulkDeleteResult *stats,
vacuum_delay_point();
+ if (info->report_parallel_progress && (blkno % REPORT_PARALLEL_VACUUM_EVERY_PAGES) == 0)
+ parallel_vacuum_update_progress(info->parallel_vacuum_state);
+
for (i = 0; i < nRoot; i++)
{
ginVacuumPostingTree(&gvs, rootOfPostingTree[i]);
@@ -751,6 +754,9 @@ ginvacuumcleanup(IndexVacuumInfo *info, IndexBulkDeleteResult *stats)
vacuum_delay_point();
+ if (info->report_parallel_progress && (blkno % REPORT_PARALLEL_VACUUM_EVERY_PAGES) == 0)
+ parallel_vacuum_update_progress(info->parallel_vacuum_state);
+
buffer = ReadBufferExtended(index, MAIN_FORKNUM, blkno,
RBM_NORMAL, info->strategy);
LockBuffer(buffer, GIN_SHARE);
diff --git a/src/backend/access/gist/gistvacuum.c b/src/backend/access/gist/gistvacuum.c
index 0aa6e58a62..d99c92c0ae 100644
--- a/src/backend/access/gist/gistvacuum.c
+++ b/src/backend/access/gist/gistvacuum.c
@@ -223,7 +223,11 @@ gistvacuumscan(IndexVacuumInfo *info, IndexBulkDeleteResult *stats,
break;
/* Iterate over pages, then loop back to recheck length */
for (; blkno < num_pages; blkno++)
+ {
gistvacuumpage(&vstate, blkno, blkno);
+ if (info->report_parallel_progress && (blkno % REPORT_PARALLEL_VACUUM_EVERY_PAGES) == 0)
+ parallel_vacuum_update_progress(info->parallel_vacuum_state);
+ }
}
/*
diff --git a/src/backend/access/hash/hash.c b/src/backend/access/hash/hash.c
index 77fd147f68..e8c6df1d5f 100644
--- a/src/backend/access/hash/hash.c
+++ b/src/backend/access/hash/hash.c
@@ -505,6 +505,12 @@ loop_top:
blkno = bucket_blkno;
+ /*
+ * For hash indexes, we report parallel vacuum progress
+ * for every bucket.
+ */
+ if (info->report_parallel_progress)
+ parallel_vacuum_update_progress(info->parallel_vacuum_state);
/*
* We need to acquire a cleanup lock on the primary bucket page to out
* wait concurrent scans before deleting the dead tuples.
diff --git a/src/backend/access/heap/vacuumlazy.c b/src/backend/access/heap/vacuumlazy.c
index d59711b7ec..81d1b7260a 100644
--- a/src/backend/access/heap/vacuumlazy.c
+++ b/src/backend/access/heap/vacuumlazy.c
@@ -459,6 +459,10 @@ heap_vacuum_rel(Relation rel, VacuumParams *params,
Assert(params->index_cleanup == VACOPTVALUE_AUTO);
}
+ /* report number of indexes to vacuum, if we are told to cleanup indexes */
+ if (vacrel->do_index_cleanup)
+ pgstat_progress_update_param(PROGRESS_VACUUM_INDEX_TOTAL, vacrel->nindexes);
+
vacrel->bstrategy = bstrategy;
vacrel->relfrozenxid = rel->rd_rel->relfrozenxid;
vacrel->relminmxid = rel->rd_rel->relminmxid;
@@ -2301,6 +2305,12 @@ lazy_vacuum_all_indexes(LVRelState *vacrel)
lazy_vacuum_one_index(indrel, istat, vacrel->old_live_tuples,
vacrel);
+ /*
+ * Done vacuuming an index. Increment the indexes completed
+ */
+ pgstat_progress_update_param(PROGRESS_VACUUM_INDEX_COMPLETED,
+ idx + 1);
+
if (lazy_check_wraparound_failsafe(vacrel))
{
/* Wraparound emergency -- end current index scan */
@@ -2335,11 +2345,14 @@ lazy_vacuum_all_indexes(LVRelState *vacrel)
Assert(allindexes || vacrel->failsafe_active);
/*
- * Increase and report the number of index scans.
+ * Reset and report the number of indexes scanned.
+ * Also, increase and report the number of index
+ * scans.
*
* We deliberately include the case where we started a round of bulk
* deletes that we weren't able to finish due to the failsafe triggering.
*/
+ pgstat_progress_update_param(PROGRESS_VACUUM_INDEX_COMPLETED, 0);
vacrel->num_index_scans++;
pgstat_progress_update_param(PROGRESS_VACUUM_NUM_INDEX_VACUUMS,
vacrel->num_index_scans);
@@ -2593,10 +2606,17 @@ lazy_check_wraparound_failsafe(LVRelState *vacrel)
{
vacrel->failsafe_active = true;
- /* Disable index vacuuming, index cleanup, and heap rel truncation */
+ /*
+ * Disable index vacuuming, index cleanup, and heap rel truncation
+ *
+ * Also, report to progress.h that we are no longer tracking
+ * index vacuum/cleanup.
+ */
vacrel->do_index_vacuuming = false;
vacrel->do_index_cleanup = false;
vacrel->do_rel_truncate = false;
+ pgstat_progress_update_param(PROGRESS_VACUUM_INDEX_TOTAL, 0);
+ pgstat_progress_update_param(PROGRESS_VACUUM_INDEX_COMPLETED, 0);
ereport(WARNING,
(errmsg("bypassing nonessential maintenance of table \"%s.%s.%s\" as a failsafe after %d index scans",
@@ -2644,6 +2664,12 @@ lazy_cleanup_all_indexes(LVRelState *vacrel)
vacrel->indstats[idx] =
lazy_cleanup_one_index(indrel, istat, reltuples,
estimated_count, vacrel);
+
+ /*
+ * Done cleaning an index. Increment the indexes completed
+ */
+ pgstat_progress_update_param(PROGRESS_VACUUM_INDEX_COMPLETED,
+ idx + 1);
}
}
else
@@ -2678,10 +2704,12 @@ lazy_vacuum_one_index(Relation indrel, IndexBulkDeleteResult *istat,
ivinfo.index = indrel;
ivinfo.analyze_only = false;
ivinfo.report_progress = false;
+ ivinfo.report_parallel_progress = false;
ivinfo.estimated_count = true;
ivinfo.message_level = DEBUG2;
ivinfo.num_heap_tuples = reltuples;
ivinfo.strategy = vacrel->bstrategy;
+ ivinfo.parallel_vacuum_state = NULL;
/*
* Update error traceback information.
@@ -2726,11 +2754,13 @@ lazy_cleanup_one_index(Relation indrel, IndexBulkDeleteResult *istat,
ivinfo.index = indrel;
ivinfo.analyze_only = false;
ivinfo.report_progress = false;
+ ivinfo.report_parallel_progress = false;
ivinfo.estimated_count = estimated_count;
ivinfo.message_level = DEBUG2;
ivinfo.num_heap_tuples = reltuples;
ivinfo.strategy = vacrel->bstrategy;
+ ivinfo.parallel_vacuum_state = NULL;
/*
* Update error traceback information.
diff --git a/src/backend/access/nbtree/nbtree.c b/src/backend/access/nbtree/nbtree.c
index b52eca8f38..30d4677808 100644
--- a/src/backend/access/nbtree/nbtree.c
+++ b/src/backend/access/nbtree/nbtree.c
@@ -998,6 +998,8 @@ btvacuumscan(IndexVacuumInfo *info, IndexBulkDeleteResult *stats,
if (info->report_progress)
pgstat_progress_update_param(PROGRESS_SCAN_BLOCKS_DONE,
scanblkno);
+ if (info->report_parallel_progress && (scanblkno % REPORT_PARALLEL_VACUUM_EVERY_PAGES) == 0)
+ parallel_vacuum_update_progress(info->parallel_vacuum_state);
}
}
diff --git a/src/backend/access/spgist/spgvacuum.c b/src/backend/access/spgist/spgvacuum.c
index ad90b213b9..e64134a4d2 100644
--- a/src/backend/access/spgist/spgvacuum.c
+++ b/src/backend/access/spgist/spgvacuum.c
@@ -843,6 +843,9 @@ spgvacuumscan(spgBulkDeleteState *bds)
/* empty the pending-list after each page */
if (bds->pendingList != NULL)
spgprocesspending(bds);
+ /* report parallel vacuum progress */
+ if (bds->info->report_parallel_progress && (blkno % REPORT_PARALLEL_VACUUM_EVERY_PAGES) == 0)
+ parallel_vacuum_update_progress(bds->info->parallel_vacuum_state);
}
}
diff --git a/src/backend/catalog/index.c b/src/backend/catalog/index.c
index 61f1d3926a..72f6a05e28 100644
--- a/src/backend/catalog/index.c
+++ b/src/backend/catalog/index.c
@@ -3348,10 +3348,12 @@ validate_index(Oid heapId, Oid indexId, Snapshot snapshot)
ivinfo.index = indexRelation;
ivinfo.analyze_only = false;
ivinfo.report_progress = true;
+ ivinfo.report_parallel_progress = false;
ivinfo.estimated_count = true;
ivinfo.message_level = DEBUG2;
ivinfo.num_heap_tuples = heapRelation->rd_rel->reltuples;
ivinfo.strategy = NULL;
+ ivinfo.parallel_vacuum_state = NULL;
/*
* Encode TIDs as int8 values for the sort, rather than directly sorting
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index 2d8104b090..c37b20b91b 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1165,7 +1165,8 @@ CREATE VIEW pg_stat_progress_vacuum AS
END AS phase,
S.param2 AS heap_blks_total, S.param3 AS heap_blks_scanned,
S.param4 AS heap_blks_vacuumed, S.param5 AS index_vacuum_count,
- S.param6 AS max_dead_tuples, S.param7 AS num_dead_tuples
+ S.param6 AS max_dead_tuples, S.param7 AS num_dead_tuples,
+ S.param8 AS indexes_total, S.param9 AS indexes_completed
FROM pg_stat_get_progress_info('VACUUM') AS S
LEFT JOIN pg_database D ON S.datid = D.oid;
diff --git a/src/backend/commands/analyze.c b/src/backend/commands/analyze.c
index da1f0f043b..15c0eea75c 100644
--- a/src/backend/commands/analyze.c
+++ b/src/backend/commands/analyze.c
@@ -712,6 +712,7 @@ do_analyze_rel(Relation onerel, VacuumParams *params,
ivinfo.message_level = elevel;
ivinfo.num_heap_tuples = onerel->rd_rel->reltuples;
ivinfo.strategy = vac_strategy;
+ ivinfo.parallel_vacuum_state = NULL;
stats = index_vacuum_cleanup(&ivinfo, NULL);
diff --git a/src/backend/commands/vacuumparallel.c b/src/backend/commands/vacuumparallel.c
index f26d796e52..c87a7f023f 100644
--- a/src/backend/commands/vacuumparallel.c
+++ b/src/backend/commands/vacuumparallel.c
@@ -30,6 +30,7 @@
#include "access/table.h"
#include "access/xact.h"
#include "catalog/index.h"
+#include "commands/progress.h"
#include "commands/vacuum.h"
#include "optimizer/paths.h"
#include "pgstat.h"
@@ -103,6 +104,17 @@ typedef struct PVShared
/* Counter for vacuuming and cleanup */
pg_atomic_uint32 idx;
+
+ /*
+ * Counter for vacuuming and cleanup progress reporting.
+ * This value is used to report index vacuum/cleanup progress
+ * in parallel_vacuum_progress_report. We keep this
+ * counter to avoid having to loop through
+ * ParallelVacuumState->indstats to determine the number
+ * of indexes completed.
+ */
+ pg_atomic_uint32 nindexes_completed;
+
} PVShared;
/* Status used during parallel index vacuum or cleanup */
@@ -213,6 +225,7 @@ static void parallel_vacuum_process_one_index(ParallelVacuumState *pvs, Relation
static bool parallel_vacuum_index_is_parallel_safe(Relation indrel, int num_index_scans,
bool vacuum);
static void parallel_vacuum_error_callback(void *arg);
+static void parallel_vacuum_wait_to_finish(ParallelVacuumState *pvs);
/*
* Try to enter parallel mode and create a parallel context. Then initialize
@@ -364,6 +377,7 @@ parallel_vacuum_init(Relation rel, Relation *indrels, int nindexes,
pg_atomic_init_u32(&(shared->cost_balance), 0);
pg_atomic_init_u32(&(shared->active_nworkers), 0);
pg_atomic_init_u32(&(shared->idx), 0);
+ pg_atomic_init_u32(&(shared->nindexes_completed), 0);
shm_toc_insert(pcxt->toc, PARALLEL_VACUUM_KEY_SHARED, shared);
pvs->shared = shared;
@@ -618,8 +632,9 @@ parallel_vacuum_process_all_indexes(ParallelVacuumState *pvs, int num_index_scan
vacuum));
}
- /* Reset the parallel index processing counter */
+ /* Reset the parallel index processing counter ( index progress counter also ) */
pg_atomic_write_u32(&(pvs->shared->idx), 0);
+ pg_atomic_write_u32(&(pvs->shared->nindexes_completed), 0);
/* Setup the shared cost-based vacuum delay and launch workers */
if (nworkers > 0)
@@ -688,7 +703,13 @@ parallel_vacuum_process_all_indexes(ParallelVacuumState *pvs, int num_index_scan
*/
if (nworkers > 0)
{
- /* Wait for all vacuum workers to finish */
+ /*
+ * Wait for all indexes to be vacuumed while
+ * updating the parallel vacuum index progress,
+ * and then wait for all workers to finish.
+ */
+ parallel_vacuum_wait_to_finish(pvs);
+
WaitForParallelWorkersToFinish(pvs->pcxt);
for (int i = 0; i < pvs->pcxt->nworkers_launched; i++)
@@ -839,9 +860,13 @@ parallel_vacuum_process_one_index(ParallelVacuumState *pvs, Relation indrel,
ivinfo.num_heap_tuples = pvs->shared->reltuples;
ivinfo.strategy = pvs->bstrategy;
+ /* Only the leader should report parallel vacuum progress */
+ ivinfo.report_parallel_progress = !IsParallelWorker();
/* Update error traceback information */
pvs->indname = pstrdup(RelationGetRelationName(indrel));
pvs->status = indstats->status;
+ /* Pass ParallelVacuumState to IndexVacuumInfo for progress reporting */
+ ivinfo.parallel_vacuum_state = pvs;
switch (indstats->status)
{
@@ -888,6 +913,12 @@ parallel_vacuum_process_one_index(ParallelVacuumState *pvs, Relation indrel,
pvs->status = PARALLEL_INDVAC_STATUS_COMPLETED;
pfree(pvs->indname);
pvs->indname = NULL;
+
+ /* Update the number of indexes completed. */
+ pg_atomic_add_fetch_u32(&(pvs->shared->nindexes_completed), 1);
+
+ if (ivinfo.report_parallel_progress)
+ parallel_vacuum_update_progress(pvs);
}
/*
@@ -1072,3 +1103,42 @@ parallel_vacuum_error_callback(void *arg)
return;
}
}
+
+/*
+ * Read pvs->shared->nindexes_completed and update progress.h
+ * with indexes vacuumed so far. This function is called periodically
+ * by index AMs as well as parallel_vacuum_process_one_index.
+ *
+ * Note: This function should be used by the leader process only,
+ * and it's up to the caller to ensure this.
+ */
+void
+parallel_vacuum_update_progress(ParallelVacuumState *pvs)
+{
+ Assert(!IsParallelWorker);
+
+ if (pvs)
+ pgstat_progress_update_param(PROGRESS_VACUUM_INDEX_COMPLETED,
+ pg_atomic_read_u32(&pvs->shared->nindexes_completed));
+}
+
+/*
+ * Waiting for all indexes to be vacuumed while updating the
+ * parallel index vacuum progress.
+ */
+void
+parallel_vacuum_wait_to_finish(ParallelVacuumState *pvs)
+{
+ Assert(!IsParallelWorker);
+
+ while (pg_atomic_read_u32(&pvs->shared->nindexes_completed) < pvs->nindexes)
+ {
+ CHECK_FOR_INTERRUPTS();
+
+ parallel_vacuum_update_progress(pvs);
+
+ (void) WaitLatch(MyLatch, WL_TIMEOUT | WL_LATCH_SET | WL_EXIT_ON_PM_DEATH, 1000L,
+ WAIT_EVENT_PARALLEL_VACUUM_FINISH);
+ ResetLatch(MyLatch);
+ }
+}
diff --git a/src/backend/utils/activity/wait_event.c b/src/backend/utils/activity/wait_event.c
index b2abd75ddb..eeba2bea5a 100644
--- a/src/backend/utils/activity/wait_event.c
+++ b/src/backend/utils/activity/wait_event.c
@@ -460,6 +460,9 @@ pgstat_get_wait_ipc(WaitEventIPC w)
case WAIT_EVENT_XACT_GROUP_UPDATE:
event_name = "XactGroupUpdate";
break;
+ case WAIT_EVENT_PARALLEL_VACUUM_FINISH:
+ event_name = "ParallelVacuumFinish";
+ break;
/* no default case, so that compiler will warn */
}
diff --git a/src/include/access/genam.h b/src/include/access/genam.h
index e1c4fdbd03..8b0db91874 100644
--- a/src/include/access/genam.h
+++ b/src/include/access/genam.h
@@ -21,8 +21,9 @@
#include "utils/relcache.h"
#include "utils/snapshot.h"
-/* We don't want this file to depend on execnodes.h. */
+/* We don't want this file to depend on execnodes.h or vacuum.h. */
struct IndexInfo;
+struct ParallelVacuumState;
/*
* Struct for statistics returned by ambuild
@@ -46,10 +47,12 @@ typedef struct IndexVacuumInfo
Relation index; /* the index being vacuumed */
bool analyze_only; /* ANALYZE (without any actual vacuum) */
bool report_progress; /* emit progress.h status reports */
+ bool report_parallel_progress; /* emit progress.h status reports for parallel vacuum */
bool estimated_count; /* num_heap_tuples is an estimate */
int message_level; /* ereport level for progress messages */
double num_heap_tuples; /* tuples remaining in heap */
BufferAccessStrategy strategy; /* access strategy for reads */
+ struct ParallelVacuumState *parallel_vacuum_state; /* access parallel vacuum state for progress reporting */
} IndexVacuumInfo;
/*
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index a28938caf4..0e97c6d4ef 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -25,6 +25,8 @@
#define PROGRESS_VACUUM_NUM_INDEX_VACUUMS 4
#define PROGRESS_VACUUM_MAX_DEAD_TUPLES 5
#define PROGRESS_VACUUM_NUM_DEAD_TUPLES 6
+#define PROGRESS_VACUUM_INDEX_TOTAL 7
+#define PROGRESS_VACUUM_INDEX_COMPLETED 8
/* Phases of vacuum (as advertised via PROGRESS_VACUUM_PHASE) */
#define PROGRESS_VACUUM_PHASE_SCAN_HEAP 1
diff --git a/src/include/commands/vacuum.h b/src/include/commands/vacuum.h
index 4e4bc26a8b..fbd51786e0 100644
--- a/src/include/commands/vacuum.h
+++ b/src/include/commands/vacuum.h
@@ -64,6 +64,12 @@
/* value for checking vacuum flags */
#define VACUUM_OPTION_MAX_VALID_VALUE ((1 << 3) - 1)
+/*
+ * Parallel Index vacuum progress is reported every 1GB of blocks
+ * scanned.
+ */
+#define REPORT_PARALLEL_VACUUM_EVERY_PAGES ((BlockNumber) (((uint64) 1024 * 1024 * 1024) / BLCKSZ))
+
/* Abstract type for parallel vacuum state */
typedef struct ParallelVacuumState ParallelVacuumState;
@@ -333,5 +339,6 @@ extern bool std_typanalyze(VacAttrStats *stats);
extern double anl_random_fract(void);
extern double anl_init_selection_state(int n);
extern double anl_get_next_S(double t, int n, double *stateptr);
+extern void parallel_vacuum_update_progress(ParallelVacuumState *pvs);
#endif /* VACUUM_H */
diff --git a/src/include/utils/wait_event.h b/src/include/utils/wait_event.h
index 0b2100be4a..3a15460e98 100644
--- a/src/include/utils/wait_event.h
+++ b/src/include/utils/wait_event.h
@@ -114,6 +114,7 @@ typedef enum
WAIT_EVENT_PARALLEL_BITMAP_SCAN,
WAIT_EVENT_PARALLEL_CREATE_INDEX_SCAN,
WAIT_EVENT_PARALLEL_FINISH,
+ WAIT_EVENT_PARALLEL_VACUUM_FINISH,
WAIT_EVENT_PROCARRAY_GROUP_UPDATE,
WAIT_EVENT_PROC_SIGNAL_BARRIER,
WAIT_EVENT_PROMOTE,
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index fb9f936d43..df6f230715 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -2021,7 +2021,9 @@ pg_stat_progress_vacuum| SELECT s.pid,
s.param4 AS heap_blks_vacuumed,
s.param5 AS index_vacuum_count,
s.param6 AS max_dead_tuples,
- s.param7 AS num_dead_tuples
+ s.param7 AS num_dead_tuples,
+ s.param8 AS indexes_total,
+ s.param9 AS indexes_completed
FROM (pg_stat_get_progress_info('VACUUM'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
LEFT JOIN pg_database d ON ((s.datid = d.oid)));
pg_stat_recovery_prefetch| SELECT s.stats_reset,
--
2.32.1 (Apple Git-133)
^ permalink raw reply [nested|flat] 44+ messages in thread
* Re: Add index scan progress to pg_stat_progress_vacuum
@ 2022-12-30 18:39 Nathan Bossart <[email protected]>
parent: Imseih (AWS), Sami <[email protected]>
0 siblings, 0 replies; 44+ messages in thread
From: Nathan Bossart @ 2022-12-30 18:39 UTC (permalink / raw)
To: Imseih (AWS), Sami <[email protected]>; +Cc: Masahiko Sawada <[email protected]>; Andres Freund <[email protected]>; Robert Haas <[email protected]>; Bossart, Nathan <[email protected]>; Peter Geoghegan <[email protected]>; Justin Pryzby <[email protected]>; pgsql-hackers
On Wed, Dec 14, 2022 at 05:09:46AM +0000, Imseih (AWS), Sami wrote:
> Attached version addresses the above and the previous comments.
cfbot is complaining that this patch no longer applies. Sami, would you
mind rebasing it?
--
Nathan Bossart
Amazon Web Services: https://aws.amazon.com
^ permalink raw reply [nested|flat] 44+ messages in thread
end of thread, other threads:[~2022-12-30 18:39 UTC | newest]
Thread overview: 44+ messages (download: mbox mbox.gz follow: Atom feed)
-- links below jump to the message on this page --
2021-12-01 19:32 Add index scan progress to pg_stat_progress_vacuum Imseih (AWS), Sami <[email protected]>
2021-12-15 22:09 ` Bossart, Nathan <[email protected]>
2021-12-16 21:37 ` Imseih (AWS), Sami <[email protected]>
2021-12-16 22:03 ` Greg Stark <[email protected]>
2021-12-20 17:55 ` Imseih (AWS), Sami <[email protected]>
2021-12-20 18:37 ` Peter Geoghegan <[email protected]>
2021-12-23 08:44 ` Masahiko Sawada <[email protected]>
2021-12-28 00:13 ` Imseih (AWS), Sami <[email protected]>
2021-12-29 16:44 ` Imseih (AWS), Sami <[email protected]>
2021-12-29 17:51 ` Justin Pryzby <[email protected]>
2021-12-31 05:59 ` Imseih (AWS), Sami <[email protected]>
2021-12-20 19:05 ` Peter Geoghegan <[email protected]>
2021-12-23 10:49 ` Andrey Lepikhov <[email protected]>
2021-12-20 19:27 ` Justin Pryzby <[email protected]>
2021-12-27 17:59 ` Justin Pryzby <[email protected]>
2021-12-20 17:55 [PATCH] Add index scan progress to pg_stat_progress_vacuum Imseih (AWS), Sami <[email protected]>
2022-03-25 14:54 Re: Add index scan progress to pg_stat_progress_vacuum Masahiko Sawada <[email protected]>
2022-03-29 12:08 ` Imseih (AWS), Sami <[email protected]>
2022-03-29 12:25 ` Imseih (AWS), Sami <[email protected]>
2022-05-05 19:26 ` Imseih (AWS), Sami <[email protected]>
2022-05-26 13:41 ` Imseih (AWS), Sami <[email protected]>
2022-05-26 15:43 ` Masahiko Sawada <[email protected]>
2022-05-27 01:52 ` Imseih (AWS), Sami <[email protected]>
2022-06-03 05:39 ` Masahiko Sawada <[email protected]>
2022-06-06 14:41 ` Robert Haas <[email protected]>
2022-06-20 06:35 ` Masahiko Sawada <[email protected]>
2022-08-02 18:06 ` Jacob Champion <[email protected]>
2022-10-11 13:50 ` Imseih (AWS), Sami <[email protected]>
2022-10-12 07:15 ` Masahiko Sawada <[email protected]>
2022-10-14 20:05 ` Imseih (AWS), Sami <[email protected]>
2022-11-02 16:52 ` Imseih (AWS), Sami <[email protected]>
2022-11-03 08:16 ` Ian Lawrence Barwick <[email protected]>
2022-11-04 13:27 ` Imseih (AWS), Sami <[email protected]>
2022-11-09 03:00 ` Andres Freund <[email protected]>
2022-11-11 19:10 ` Imseih (AWS), Sami <[email protected]>
2022-11-18 13:07 ` Masahiko Sawada <[email protected]>
2022-11-28 23:57 ` Imseih (AWS), Sami <[email protected]>
2022-12-06 03:18 ` Masahiko Sawada <[email protected]>
2022-12-13 04:40 ` Imseih (AWS), Sami <[email protected]>
2022-12-14 01:43 ` Masahiko Sawada <[email protected]>
2022-12-14 05:09 ` Imseih (AWS), Sami <[email protected]>
2022-12-30 18:39 ` Nathan Bossart <[email protected]>
2022-11-08 07:49 ` Masahiko Sawada <[email protected]>
2022-10-10 16:40 Re: Add index scan progress to pg_stat_progress_vacuum Imseih (AWS), Sami <[email protected]>
This inbox is served by agora; see mirroring instructions
for how to clone and mirror all data and code used for this inbox