From fe6b1cc600f9945a2aaeabe11123ed900ac6077d Mon Sep 17 00:00:00 2001 From: Alena Rybakina Date: Mon, 16 Dec 2024 12:06:17 +0300 Subject: [PATCH 2/4] Machinery for grabbing an extended vacuum statistics on index relations. They are gathered separatelly from table statistics. As for tables, we gather vacuum shared buffers statistics for index relations like value of total_blks_hit, total_blks_read, total_blks_dirtied, wal statistics, io time during flushing buffer pages to disk, delay and total time. Due to the fact taht such statistics are common as for tables, as for indexes we set them in the union ExtVacReport structure. We only added some determination 'type' field to highlight what kind belong to these statistics: PGSTAT_EXTVAC_TABLE or PGSTAT_EXTVAC_INDEX. Generally, PGSTAT_EXTVAC_INVALID type leads to wrong code process. Some statistics belong only one type of both tables or indexes. So, we added substructures table and index inside ExtVacReport structure. Therefore, we gather only for tables such statistics like number of scanned, removed pages, their charecteristics according VM (all-visible and frozen). In addition, for tables we gather number frozen, deleted and recently dead tuples and how many times vacuum processed indexes for tables. Controversally for indexes we gather number of deleted pages and deleted tuples only. As for tables, deleted pages and deleted tuples reflect the overall performance of the vacuum for the index relationship Since vacuum clears tuple index references before clearing table tuples, we need to subtract number of collected index vacuum statistics for general statistics taken into account for the table and for the index, especially for shared buffers: total_blks_read, total_blks_hit, total_blks_dirtied, total_blks_written; wal statistics: wal_bytes, wal_fpi, wal_records; IO time: blk_read_time, blk_write_time; total time and delay time. This is necessary to take into account vacuum statistics for tables and indexes separately. Due to the fact that the vacuum can produce workers to process indexes of the table and workers store their wal and buffer statistic information separately from the place that leader does we need to store an information about ParallelWorkerNumber that's why we added the variable id_parallel_worker in the index's statistic structure. PVIndStats stryucture store statistic information for every index whether it was processed by leader or worker, so we are sure that id_parallel_worker will be updated correctly. Authors: Alena Rybakina , Andrei Lepikhov , Andrei Zubkov Reviewed-by: Dilip Kumar , Masahiko Sawada , Ilia Evdokimov , jian he , Kirill Reshke , Alexander Korotkov , Jim Nasby --- src/backend/access/heap/vacuumlazy.c | 166 ++++++++++++---- src/backend/catalog/system_views.sql | 32 +++ src/backend/commands/vacuumparallel.c | 59 +++++- src/backend/utils/activity/pgstat.c | 4 + src/backend/utils/activity/pgstat_relation.c | 52 +++-- src/backend/utils/adt/pgstatfuncs.c | 133 +++++++++++-- src/backend/utils/misc/guc_tables.c | 2 +- src/include/catalog/pg_proc.dat | 9 + src/include/commands/vacuum.h | 26 +++ src/include/pgstat.h | 61 ++++-- src/include/utils/pgstat_internal.h | 1 - src/test/regress/expected/rules.out | 22 +++ .../expected/vacuum_index_statistics.out | 183 ++++++++++++++++++ .../expected/vacuum_tables_statistics.out | 40 +++- src/test/regress/parallel_schedule | 1 + .../regress/sql/vacuum_index_statistics.sql | 151 +++++++++++++++ .../regress/sql/vacuum_tables_statistics.sql | 12 ++ 17 files changed, 864 insertions(+), 90 deletions(-) create mode 100644 src/test/regress/expected/vacuum_index_statistics.out create mode 100644 src/test/regress/sql/vacuum_index_statistics.sql diff --git a/src/backend/access/heap/vacuumlazy.c b/src/backend/access/heap/vacuumlazy.c index 919f0103e85..15d27ed39d9 100644 --- a/src/backend/access/heap/vacuumlazy.c +++ b/src/backend/access/heap/vacuumlazy.c @@ -166,6 +166,7 @@ typedef struct LVRelState char *dbname; char *relnamespace; Oid reloid; + Oid indoid; char *relname; char *indname; /* Current index name */ BlockNumber blkno; /* used only for heap operations */ @@ -242,19 +243,6 @@ typedef struct LVSavedErrInfo VacErrPhase phase; } LVSavedErrInfo; -/* - * Counters and usage data for extended stats tracking. - */ -typedef struct LVExtStatCounters -{ - TimestampTz starttime; - WalUsage walusage; - BufferUsage bufusage; - double VacuumDelayTime; - PgStat_Counter blocks_fetched; - PgStat_Counter blocks_hit; -} LVExtStatCounters; - /* non-export function prototypes */ static void lazy_scan_heap(LVRelState *vacrel); static bool heap_vac_scan_next_block(LVRelState *vacrel, BlockNumber *blkno, @@ -352,7 +340,8 @@ extvac_stats_start(Relation rel, LVExtStatCounters *counters) */ static void extvac_stats_end(Relation rel, LVExtStatCounters *counters, - ExtVacReport *report) + ExtVacReport *report, BufferUsage *worker_bufferusage, + WalUsage *worker_walusage) { WalUsage walusage; BufferUsage bufusage; @@ -365,10 +354,16 @@ extvac_stats_end(Relation rel, LVExtStatCounters *counters, /* Calculate diffs of global stat parameters on WAL and buffer usage. */ memset(&walusage, 0, sizeof(WalUsage)); - WalUsageAccumDiff(&walusage, &pgWalUsage, &counters->walusage); + if(worker_walusage == NULL) + WalUsageAccumDiff(&walusage, &pgWalUsage, &counters->walusage); + else + WalUsageAccumDiff(&walusage, worker_walusage, &counters->walusage); memset(&bufusage, 0, sizeof(BufferUsage)); - BufferUsageAccumDiff(&bufusage, &pgBufferUsage, &counters->bufusage); + if(worker_bufferusage == NULL) + BufferUsageAccumDiff(&bufusage, &pgBufferUsage, &counters->bufusage); + else + BufferUsageAccumDiff(&bufusage, worker_bufferusage, &counters->bufusage); endtime = GetCurrentTimestamp(); TimestampDifference(counters->starttime, endtime, &secs, &usecs); @@ -408,6 +403,59 @@ extvac_stats_end(Relation rel, LVExtStatCounters *counters, rel->pgstat_info->counts.blocks_hit - counters->blocks_hit; } +void +extvac_stats_start_idx(Relation rel, IndexBulkDeleteResult *stats, + LVExtStatCountersIdx *counters, + BufferUsage *buffusage, WalUsage *walusage) +{ + if(!pgstat_track_vacuum_statistics) + return; + + extvac_stats_start(rel, &counters->common); + counters->pages_deleted = counters->tuples_removed = 0; + + if(buffusage != NULL) + memset(&(counters->common.bufusage), 0, sizeof(BufferUsage)); + + if(walusage != NULL) + memset(&(counters->common.walusage), 0, sizeof(WalUsage)); + + + if (stats != NULL) + { + /* + * XXX: Why do we need this code here? If it is needed, I feel lack of + * comments, describing the reason. + */ + counters->tuples_removed = stats->tuples_removed; + counters->pages_deleted = stats->pages_deleted; + } +} + +void +extvac_stats_end_idx(Relation rel, IndexBulkDeleteResult *stats, + LVExtStatCountersIdx *counters, ExtVacReport *report, + BufferUsage *buffusage, WalUsage *walusage) +{ + extvac_stats_end(rel, &counters->common, report, + buffusage, walusage); + report->type = PGSTAT_EXTVAC_INDEX; + + if (stats != NULL) + { + /* + * if something goes wrong or an user doesn't want to track a database + * activity - just suppress it. + */ + + /* Fill index-specific extended stats fields */ + report->tuples_deleted = + stats->tuples_removed - counters->tuples_removed; + report->index.pages_deleted = + stats->pages_deleted - counters->pages_deleted; + } +} + /* * heap_vacuum_rel() -- perform VACUUM for one heap relation * @@ -442,11 +490,6 @@ heap_vacuum_rel(Relation rel, VacuumParams *params, LVExtStatCounters extVacCounters; ExtVacReport extVacReport; char **indnames = NULL; - ExtVacReport allzero; - - /* Initialize vacuum statistics */ - memset(&allzero, 0, sizeof(ExtVacReport)); - extVacReport = allzero; verbose = (params->options & VACOPT_VERBOSE) != 0; instrument = (verbose || (AmAutoVacuumWorkerProcess() && @@ -714,25 +757,36 @@ heap_vacuum_rel(Relation rel, VacuumParams *params, new_rel_allvisible, vacrel->nindexes > 0, vacrel->NewRelfrozenXid, vacrel->NewRelminMxid, &frozenxid_updated, &minmulti_updated, false); - - /* Make generic extended vacuum stats report */ - extvac_stats_end(rel, &extVacCounters, &extVacReport); + extvac_stats_end(rel, &extVacCounters, &extVacReport, NULL, NULL); if(pgstat_track_vacuum_statistics) { /* Fill heap-specific extended stats fields */ - extVacReport.pages_scanned = vacrel->scanned_pages; - extVacReport.pages_removed = vacrel->removed_pages; - extVacReport.vm_new_frozen_pages = vacrel->vm_new_frozen_pages; - extVacReport.vm_new_visible_pages = vacrel->vm_new_visible_pages; - extVacReport.vm_new_visible_frozen_pages = vacrel->vm_new_visible_frozen_pages; + extVacReport.type = PGSTAT_EXTVAC_TABLE; + extVacReport.table.pages_scanned = vacrel->scanned_pages; + extVacReport.table.pages_removed = vacrel->removed_pages; + extVacReport.table.vm_new_frozen_pages = vacrel->vm_new_frozen_pages; + extVacReport.table.vm_new_visible_pages = vacrel->vm_new_visible_pages; + extVacReport.table.vm_new_visible_frozen_pages = vacrel->vm_new_visible_frozen_pages; extVacReport.tuples_deleted = vacrel->tuples_deleted; - extVacReport.tuples_frozen = vacrel->tuples_frozen; - extVacReport.recently_dead_tuples = vacrel->recently_dead_tuples; - extVacReport.missed_dead_tuples = vacrel->missed_dead_tuples; - extVacReport.missed_dead_pages = vacrel->missed_dead_pages; - extVacReport.index_vacuum_count = vacrel->num_index_scans; - extVacReport.wraparound_failsafe_count = vacrel->wraparound_failsafe_count; + extVacReport.table.tuples_frozen = vacrel->tuples_frozen; + extVacReport.table.recently_dead_tuples = vacrel->recently_dead_tuples; + extVacReport.table.recently_dead_tuples = vacrel->recently_dead_tuples; + extVacReport.table.missed_dead_tuples = vacrel->missed_dead_tuples; + extVacReport.table.missed_dead_pages = vacrel->missed_dead_pages; + extVacReport.table.index_vacuum_count = vacrel->num_index_scans; + extVacReport.table.wraparound_failsafe_count = vacrel->wraparound_failsafe_count; + + } + else + { + ExtVacReport allzero; + + /* Initialize vacuum statistics with 0 values to prevent + * adding garbage values ​​in memory + */ + memset(&allzero, 0, sizeof(ExtVacReport)); + extVacReport = allzero; } /* @@ -2507,6 +2561,7 @@ lazy_vacuum_heap_page(LVRelState *vacrel, BlockNumber blkno, Buffer buffer, { Assert(!TransactionIdIsValid(visibility_cutoff_xid)); flags |= VISIBILITYMAP_ALL_FROZEN; + vacrel->vm_new_frozen_pages++; } PageSetAllVisible(page); @@ -2675,6 +2730,10 @@ lazy_vacuum_one_index(Relation indrel, IndexBulkDeleteResult *istat, { IndexVacuumInfo ivinfo; LVSavedErrInfo saved_err_info; + LVExtStatCountersIdx extVacCounters; + ExtVacReport extVacReport; + + extvac_stats_start_idx(indrel, istat, &extVacCounters, NULL, NULL); ivinfo.index = indrel; ivinfo.heaprel = vacrel->rel; @@ -2693,6 +2752,7 @@ lazy_vacuum_one_index(Relation indrel, IndexBulkDeleteResult *istat, */ Assert(vacrel->indname == NULL); vacrel->indname = pstrdup(RelationGetRelationName(indrel)); + vacrel->indoid = RelationGetRelid(indrel); update_vacuum_error_info(vacrel, &saved_err_info, VACUUM_ERRCB_PHASE_VACUUM_INDEX, InvalidBlockNumber, InvalidOffsetNumber); @@ -2701,6 +2761,15 @@ lazy_vacuum_one_index(Relation indrel, IndexBulkDeleteResult *istat, istat = vac_bulkdel_one_index(&ivinfo, istat, vacrel->dead_items, vacrel->dead_items_info); + if(pgstat_track_vacuum_statistics) + { + /* Make extended vacuum stats report for index */ + extvac_stats_end_idx(indrel, istat, &extVacCounters, &extVacReport, NULL, NULL); + pgstat_report_vacuum(RelationGetRelid(indrel), + indrel->rd_rel->relisshared, + 0, 0, &extVacReport); + } + /* Revert to the previous phase information for error traceback */ restore_vacuum_error_info(vacrel, &saved_err_info); pfree(vacrel->indname); @@ -2725,6 +2794,10 @@ lazy_cleanup_one_index(Relation indrel, IndexBulkDeleteResult *istat, { IndexVacuumInfo ivinfo; LVSavedErrInfo saved_err_info; + LVExtStatCountersIdx extVacCounters; + ExtVacReport extVacReport; + + extvac_stats_start_idx(indrel, istat, &extVacCounters, NULL, NULL); ivinfo.index = indrel; ivinfo.heaprel = vacrel->rel; @@ -2744,12 +2817,22 @@ lazy_cleanup_one_index(Relation indrel, IndexBulkDeleteResult *istat, */ Assert(vacrel->indname == NULL); vacrel->indname = pstrdup(RelationGetRelationName(indrel)); + vacrel->indoid = RelationGetRelid(indrel); update_vacuum_error_info(vacrel, &saved_err_info, VACUUM_ERRCB_PHASE_INDEX_CLEANUP, InvalidBlockNumber, InvalidOffsetNumber); istat = vac_cleanup_one_index(&ivinfo, istat); + if(pgstat_track_vacuum_statistics) + { + /* Make extended vacuum stats report for index */ + extvac_stats_end_idx(indrel, istat, &extVacCounters, &extVacReport, NULL, NULL); + pgstat_report_vacuum(RelationGetRelid(indrel), + indrel->rd_rel->relisshared, + 0, 0, &extVacReport); + } + /* Revert to the previous phase information for error traceback */ restore_vacuum_error_info(vacrel, &saved_err_info); pfree(vacrel->indname); @@ -3325,6 +3408,8 @@ update_relstats_all_indexes(LVRelState *vacrel) Relation *indrels = vacrel->indrels; int nindexes = vacrel->nindexes; IndexBulkDeleteResult **indstats = vacrel->indstats; + LVExtStatCountersIdx extVacCounters; + ExtVacReport extVacReport; Assert(vacrel->do_index_cleanup); @@ -3336,6 +3421,8 @@ update_relstats_all_indexes(LVRelState *vacrel) if (istat == NULL || istat->estimated_count) continue; + extvac_stats_start_idx(indrel, istat, &extVacCounters, NULL, NULL); + /* Update index statistics */ vac_update_relstats(indrel, istat->num_pages, @@ -3345,6 +3432,15 @@ update_relstats_all_indexes(LVRelState *vacrel) InvalidTransactionId, InvalidMultiXactId, NULL, NULL, false); + + if(pgstat_track_vacuum_statistics) + { + extvac_stats_end_idx(indrel, istat, &extVacCounters, &extVacReport, NULL, NULL); + + pgstat_report_vacuum(RelationGetRelid(indrel), + indrel->rd_rel->relisshared, + 0, 0, &extVacReport); + } } } diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index 43ac27ed5b4..09e93b21e82 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1431,3 +1431,35 @@ FROM pg_class rel JOIN pg_namespace ns ON ns.oid = rel.relnamespace, LATERAL pg_stat_get_vacuum_tables(rel.oid) stats WHERE rel.relkind = 'r'; + +CREATE VIEW pg_stat_vacuum_indexes AS +SELECT + rel.oid as relid, + ns.nspname AS schemaname, + rel.relname AS relname, + + total_blks_read AS total_blks_read, + total_blks_hit AS total_blks_hit, + total_blks_dirtied AS total_blks_dirtied, + total_blks_written AS total_blks_written, + + rel_blks_read AS rel_blks_read, + rel_blks_hit AS rel_blks_hit, + + pages_deleted AS pages_deleted, + tuples_deleted AS tuples_deleted, + + wal_records AS wal_records, + wal_fpi AS wal_fpi, + wal_bytes AS wal_bytes, + + blk_read_time AS blk_read_time, + blk_write_time AS blk_write_time, + + delay_time AS delay_time, + total_time AS total_time +FROM + pg_class rel + JOIN pg_namespace ns ON ns.oid = rel.relnamespace, + LATERAL pg_stat_get_vacuum_indexes(rel.oid) stats +WHERE rel.relkind = 'i'; \ No newline at end of file diff --git a/src/backend/commands/vacuumparallel.c b/src/backend/commands/vacuumparallel.c index 672f8f4bfe8..c6ee0d63d13 100644 --- a/src/backend/commands/vacuumparallel.c +++ b/src/backend/commands/vacuumparallel.c @@ -154,6 +154,10 @@ typedef struct PVIndStats */ bool istat_updated; /* are the stats updated? */ IndexBulkDeleteResult istat; + + LVExtStatCountersIdx counters; + ExtVacReport idx_report; + int id_parallel_worker; /* detect index was processed by postmster or worker */ } PVIndStats; /* @@ -654,6 +658,8 @@ parallel_vacuum_process_all_indexes(ParallelVacuumState *pvs, int num_index_scan { PVIndStats *indstats = &(pvs->indstats[i]); + indstats->id_parallel_worker = -2; + Assert(indstats->status == PARALLEL_INDVAC_STATUS_INITIAL); indstats->status = new_status; indstats->parallel_workers_can_process = @@ -661,6 +667,12 @@ parallel_vacuum_process_all_indexes(ParallelVacuumState *pvs, int num_index_scan parallel_vacuum_index_is_parallel_safe(pvs->indrels[i], num_index_scans, vacuum)); + + /* Sava buffer and wal statistics before vacuuming to track them + * for the leader. + */ + extvac_stats_start_idx(pvs->indrels[i], &(indstats->istat), + &(indstats->counters),NULL, NULL); } /* Reset the parallel index processing and progress counters */ @@ -727,19 +739,10 @@ parallel_vacuum_process_all_indexes(ParallelVacuumState *pvs, int num_index_scan */ parallel_vacuum_process_safe_indexes(pvs); - /* - * Next, accumulate buffer and WAL usage. (This must wait for the workers - * to finish, or we might get incomplete data.) - */ if (nworkers > 0) - { /* Wait for all vacuum workers to finish */ WaitForParallelWorkersToFinish(pvs->pcxt); - for (int i = 0; i < pvs->pcxt->nworkers_launched; i++) - InstrAccumParallelQuery(&pvs->buffer_usage[i], &pvs->wal_usage[i]); - } - /* * Reset all index status back to initial (while checking that we have * vacuumed all indexes). @@ -752,9 +755,44 @@ parallel_vacuum_process_all_indexes(ParallelVacuumState *pvs, int num_index_scan elog(ERROR, "parallel index vacuum on index \"%s\" is not completed", RelationGetRelationName(pvs->indrels[i])); + /* If an index was processed by worker we need to gather wal and + * buffer statistics from pvs->buffer_usage and pvs->wal_usage, + * otherwice for leader they can be detected through substract + * of global statistics of pgWalUsage and pgBufferUsage. + */ + if(pgstat_track_vacuum_statistics) + { + /* We expect that all indexes have updated it */ + Assert(indstats->id_parallel_worker != -2); + + if(indstats->id_parallel_worker == -1) + extvac_stats_end_idx(pvs->indrels[i], &(indstats->istat), &(indstats->counters), + &(indstats->idx_report), NULL, NULL); + else + { + /* We need to reset Buffer and Wal usage statistics */ + memset(&(indstats->counters.common.bufusage), 0, sizeof(BufferUsage)); + memset(&(indstats->counters.common.walusage), 0, sizeof(WalUsage)); + extvac_stats_end_idx(pvs->indrels[i], &(indstats->istat), &(indstats->counters), + &(indstats->idx_report), &pvs->buffer_usage[indstats->id_parallel_worker], &pvs->wal_usage[indstats->id_parallel_worker]); + } + + pgstat_report_vacuum(RelationGetRelid(pvs->indrels[i]), + pvs->indrels[i]->rd_rel->relisshared, + 0, 0, &(indstats->idx_report)); + } + indstats->status = PARALLEL_INDVAC_STATUS_INITIAL; } + /* + * Next, accumulate buffer and WAL usage. (This must wait for the workers + * to finish, or we might get incomplete data.) + */ + if (nworkers > 0) + for (int i = 0; i < pvs->pcxt->nworkers_launched; i++) + InstrAccumParallelQuery(&pvs->buffer_usage[i], &pvs->wal_usage[i]); + /* * Carry the shared balance value to heap scan and disable shared costing */ @@ -925,6 +963,9 @@ parallel_vacuum_process_one_index(ParallelVacuumState *pvs, Relation indrel, pfree(istat_res); } + /* Like ParallelWorkerNumber can be -1 for leader and more 0 for workers */ + indstats->id_parallel_worker = ParallelWorkerNumber; + /* * Update the status to completed. No need to lock here since each worker * touches different indexes. diff --git a/src/backend/utils/activity/pgstat.c b/src/backend/utils/activity/pgstat.c index d21b9302c29..dc81bb12c86 100644 --- a/src/backend/utils/activity/pgstat.c +++ b/src/backend/utils/activity/pgstat.c @@ -1190,6 +1190,10 @@ pgstat_build_snapshot(PgStat_Kind statKind) if (p->dropped) continue; + if (statKind != PGSTAT_KIND_INVALID && statKind != p->key.kind) + /* Load stat of specific type, if defined */ + continue; + Assert(pg_atomic_read_u32(&p->refcount) > 0); stats_data = dsa_get_address(pgStatLocal.dsa, p->body); diff --git a/src/backend/utils/activity/pgstat_relation.c b/src/backend/utils/activity/pgstat_relation.c index 458bd4ece49..db612d243cd 100644 --- a/src/backend/utils/activity/pgstat_relation.c +++ b/src/backend/utils/activity/pgstat_relation.c @@ -47,8 +47,6 @@ static void add_tabstat_xact_level(PgStat_TableStatus *pgstat_info, int nest_lev static void ensure_tabstat_xact_level(PgStat_TableStatus *pgstat_info); static void save_truncdrop_counters(PgStat_TableXactStatus *trans, bool is_drop); static void restore_truncdrop_counters(PgStat_TableXactStatus *trans); -static void pgstat_accumulate_extvac_stats(ExtVacReport *dst, ExtVacReport *src, - bool accumulate_reltype_specific_info); /* @@ -991,10 +989,13 @@ restore_truncdrop_counters(PgStat_TableXactStatus *trans) } } -static void +void pgstat_accumulate_extvac_stats(ExtVacReport *dst, ExtVacReport *src, bool accumulate_reltype_specific_info) { + if(!pgstat_track_vacuum_statistics) + return; + dst->total_blks_read += src->total_blks_read; dst->total_blks_hit += src->total_blks_hit; dst->total_blks_dirtied += src->total_blks_dirtied; @@ -1010,20 +1011,35 @@ pgstat_accumulate_extvac_stats(ExtVacReport *dst, ExtVacReport *src, if (!accumulate_reltype_specific_info) return; - dst->blks_fetched += src->blks_fetched; - dst->blks_hit += src->blks_hit; - - dst->pages_scanned += src->pages_scanned; - dst->pages_removed += src->pages_removed; - dst->vm_new_frozen_pages += src->vm_new_frozen_pages; - dst->vm_new_visible_pages += src->vm_new_visible_pages; - dst->vm_new_visible_frozen_pages += src->vm_new_visible_frozen_pages; - dst->tuples_deleted += src->tuples_deleted; - dst->tuples_frozen += src->tuples_frozen; - dst->recently_dead_tuples += src->recently_dead_tuples; - dst->index_vacuum_count += src->index_vacuum_count; - dst->wraparound_failsafe_count += src->wraparound_failsafe_count; - dst->missed_dead_pages += src->missed_dead_pages; - dst->missed_dead_tuples += src->missed_dead_tuples; + if (dst->type == PGSTAT_EXTVAC_INVALID) + dst->type = src->type; + + Assert(src->type == PGSTAT_EXTVAC_INVALID || src->type == dst->type); + + if (dst->type == src->type) + { + dst->blks_fetched += src->blks_fetched; + dst->blks_hit += src->blks_hit; + if (dst->type == PGSTAT_EXTVAC_TABLE) + { + dst->table.pages_scanned += src->table.pages_scanned; + dst->table.pages_removed += src->table.pages_removed; + dst->table.vm_new_frozen_pages += src->table.vm_new_frozen_pages; + dst->table.vm_new_visible_pages += src->table.vm_new_visible_pages; + dst->table.vm_new_visible_frozen_pages += src->table.vm_new_visible_frozen_pages; + dst->tuples_deleted += src->tuples_deleted; + dst->table.tuples_frozen += src->table.tuples_frozen; + dst->table.recently_dead_tuples += src->table.recently_dead_tuples; + dst->table.index_vacuum_count += src->table.index_vacuum_count; + dst->table.missed_dead_pages += src->table.missed_dead_pages; + dst->table.missed_dead_tuples += src->table.missed_dead_tuples; + dst->table.wraparound_failsafe_count += src->table.wraparound_failsafe_count; + } + else if (dst->type == PGSTAT_EXTVAC_INDEX) + { + dst->index.pages_deleted += src->index.pages_deleted; + dst->tuples_deleted += src->tuples_deleted; + } + } } \ No newline at end of file diff --git a/src/backend/utils/adt/pgstatfuncs.c b/src/backend/utils/adt/pgstatfuncs.c index f180ac0fa02..d41687ca39a 100644 --- a/src/backend/utils/adt/pgstatfuncs.c +++ b/src/backend/utils/adt/pgstatfuncs.c @@ -2286,18 +2286,19 @@ pg_stat_get_vacuum_tables(PG_FUNCTION_ARGS) extvacuum->blks_hit); values[i++] = Int64GetDatum(extvacuum->blks_hit); - values[i++] = Int64GetDatum(extvacuum->pages_scanned); - values[i++] = Int64GetDatum(extvacuum->pages_removed); - values[i++] = Int64GetDatum(extvacuum->vm_new_frozen_pages); - values[i++] = Int64GetDatum(extvacuum->vm_new_visible_pages); - values[i++] = Int64GetDatum(extvacuum->vm_new_visible_frozen_pages); - values[i++] = Int64GetDatum(extvacuum->missed_dead_pages); + values[i++] = Int64GetDatum(extvacuum->table.pages_scanned); + values[i++] = Int64GetDatum(extvacuum->table.pages_removed); + values[i++] = Int64GetDatum(extvacuum->table.vm_new_frozen_pages); + values[i++] = Int64GetDatum(extvacuum->table.vm_new_visible_pages); + values[i++] = Int64GetDatum(extvacuum->table.vm_new_visible_frozen_pages); + values[i++] = Int64GetDatum(extvacuum->table.missed_dead_pages); values[i++] = Int64GetDatum(extvacuum->tuples_deleted); - values[i++] = Int64GetDatum(extvacuum->tuples_frozen); - values[i++] = Int64GetDatum(extvacuum->recently_dead_tuples); - values[i++] = Int64GetDatum(extvacuum->missed_dead_tuples); - values[i++] = Int64GetDatum(extvacuum->wraparound_failsafe_count); - values[i++] = Int64GetDatum(extvacuum->index_vacuum_count); + values[i++] = Int64GetDatum(extvacuum->table.tuples_frozen); + values[i++] = Int64GetDatum(extvacuum->table.recently_dead_tuples); + values[i++] = Int64GetDatum(extvacuum->table.missed_dead_tuples); + + values[i++] = Int64GetDatum(extvacuum->table.wraparound_failsafe_count); + values[i++] = Int64GetDatum(extvacuum->table.index_vacuum_count); values[i++] = Int64GetDatum(extvacuum->wal_records); values[i++] = Int64GetDatum(extvacuum->wal_fpi); @@ -2316,6 +2317,116 @@ pg_stat_get_vacuum_tables(PG_FUNCTION_ARGS) Assert(i == PG_STAT_GET_VACUUM_TABLES_STATS_COLS); + /* Returns the record as Datum */ + PG_RETURN_DATUM(HeapTupleGetDatum(heap_form_tuple(tupdesc, values, nulls))); +} + +/* + * Get the vacuum statistics for the heap tables. + */ +Datum +pg_stat_get_vacuum_indexes(PG_FUNCTION_ARGS) +{ + #define PG_STAT_GET_VACUUM_INDEX_STATS_COLS 16 + + Oid relid = PG_GETARG_OID(0); + PgStat_StatTabEntry *tabentry; + ExtVacReport *extvacuum; + TupleDesc tupdesc; + Datum values[PG_STAT_GET_VACUUM_INDEX_STATS_COLS] = {0}; + bool nulls[PG_STAT_GET_VACUUM_INDEX_STATS_COLS] = {0}; + char buf[256]; + int i = 0; + ExtVacReport allzero; + + /* Initialise attributes information in the tuple descriptor */ + tupdesc = CreateTemplateTupleDesc(PG_STAT_GET_VACUUM_INDEX_STATS_COLS); + + TupleDescInitEntry(tupdesc, (AttrNumber) ++i, "relid", + INT4OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) ++i, "total_ blks_read", + INT8OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) ++i, "total_blks_hit", + INT8OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) ++i, "total_blks_dirtied", + INT8OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) ++i, "total_blks_written", + INT8OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) ++i, "rel_blks_read", + INT8OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) ++i, "rel_blks_hit", + INT8OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) ++i, "pages_deleted", + INT8OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) ++i, "tuples_deleted", + INT8OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) ++i, "wal_records", + INT8OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) ++i, "wal_fpi", + INT8OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) ++i, "wal_bytes", + NUMERICOID, -1, 0); + + TupleDescInitEntry(tupdesc, (AttrNumber) ++i, "blk_read_time", + FLOAT8OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) ++i, "blk_write_time", + FLOAT8OID, -1, 0); + + TupleDescInitEntry(tupdesc, (AttrNumber) ++i, "delay_time", + FLOAT8OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) ++i, "total_time", + FLOAT8OID, -1, 0); + + Assert(i == PG_STAT_GET_VACUUM_INDEX_STATS_COLS); + + BlessTupleDesc(tupdesc); + + tabentry = pgstat_fetch_stat_tabentry(relid); + + if (tabentry == NULL) + { + /* If the subscription is not found, initialise its stats */ + memset(&allzero, 0, sizeof(ExtVacReport)); + extvacuum = &allzero; + } + else + { + extvacuum = &(tabentry->vacuum_ext); + } + + i = 0; + + values[i++] = ObjectIdGetDatum(relid); + + values[i++] = Int64GetDatum(extvacuum->total_blks_read); + values[i++] = Int64GetDatum(extvacuum->total_blks_hit); + values[i++] = Int64GetDatum(extvacuum->total_blks_dirtied); + values[i++] = Int64GetDatum(extvacuum->total_blks_written); + + values[i++] = Int64GetDatum(extvacuum->blks_fetched - + extvacuum->blks_hit); + values[i++] = Int64GetDatum(extvacuum->blks_hit); + + values[i++] = Int64GetDatum(extvacuum->index.pages_deleted); + values[i++] = Int64GetDatum(extvacuum->tuples_deleted); + + values[i++] = Int64GetDatum(extvacuum->wal_records); + values[i++] = Int64GetDatum(extvacuum->wal_fpi); + + /* Convert to numeric, like pg_stat_statements */ + snprintf(buf, sizeof buf, UINT64_FORMAT, extvacuum->wal_bytes); + values[i++] = DirectFunctionCall3(numeric_in, + CStringGetDatum(buf), + ObjectIdGetDatum(0), + Int32GetDatum(-1)); + + values[i++] = Float8GetDatum(extvacuum->blk_read_time); + values[i++] = Float8GetDatum(extvacuum->blk_write_time); + values[i++] = Float8GetDatum(extvacuum->delay_time); + values[i++] = Float8GetDatum(extvacuum->total_time); + + Assert(i == PG_STAT_GET_VACUUM_INDEX_STATS_COLS); + /* Returns the record as Datum */ PG_RETURN_DATUM(HeapTupleGetDatum(heap_form_tuple(tupdesc, values, nulls))); } \ No newline at end of file diff --git a/src/backend/utils/misc/guc_tables.c b/src/backend/utils/misc/guc_tables.c index f2d31e174b4..2a96cf51a36 100644 --- a/src/backend/utils/misc/guc_tables.c +++ b/src/backend/utils/misc/guc_tables.c @@ -1480,7 +1480,7 @@ struct config_bool ConfigureNamesBool[] = }, { {"track_vacuum_statistics", PGC_SUSET, STATS_CUMULATIVE, - gettext_noop("Collects vacuum statistics for table relations."), + gettext_noop("Collects vacuum statistics for relations."), NULL }, &pgstat_track_vacuum_statistics, diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index b0e363794dc..35d6649db31 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -12447,4 +12447,13 @@ proname => 'pg_stat_get_rev_all_frozen_pages', provolatile => 's', proparallel => 'r', prorettype => 'int8', proargtypes => 'oid', prosrc => 'pg_stat_get_rev_all_frozen_pages' }, +{ oid => '8004', + descr => 'pg_stat_get_vacuum_indexes return stats values', + proname => 'pg_stat_get_vacuum_indexes', prorows => 1000, provolatile => 's', prorettype => 'record',proisstrict => 'f', + proretset => 't', + proargtypes => 'oid', + proallargtypes => '{oid,oid,int8,int8,int8,int8,int8,int8,int8,int8,int8,int8,numeric,float8,float8,float8,float8}', + proargmodes => '{i,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o}', + proargnames => '{reloid,relid,total_blks_read,total_blks_hit,total_blks_dirtied,total_blks_written,rel_blks_read,rel_blks_hit,pages_deleted,tuples_deleted,wal_records,wal_fpi,wal_bytes,blk_read_time,blk_write_time,delay_time,total_time}', + prosrc => 'pg_stat_get_vacuum_indexes' } ] diff --git a/src/include/commands/vacuum.h b/src/include/commands/vacuum.h index 94d599767df..c6f5a9ffb02 100644 --- a/src/include/commands/vacuum.h +++ b/src/include/commands/vacuum.h @@ -25,6 +25,7 @@ #include "storage/buf.h" #include "storage/lock.h" #include "utils/relcache.h" +#include "pgstat.h" /* * Flags for amparallelvacuumoptions to control the participation of bulkdelete @@ -288,6 +289,26 @@ typedef struct VacDeadItemsInfo int64 num_items; /* current # of entries */ } VacDeadItemsInfo; +/* + * Counters and usage data for extended stats tracking. + */ +typedef struct LVExtStatCounters +{ + TimestampTz starttime; + WalUsage walusage; + BufferUsage bufusage; + double VacuumDelayTime; + PgStat_Counter blocks_fetched; + PgStat_Counter blocks_hit; +} LVExtStatCounters; + +typedef struct LVExtStatCountersIdx +{ + LVExtStatCounters common; + int64 pages_deleted; + int64 tuples_removed; +} LVExtStatCountersIdx; + /* GUC parameters */ extern PGDLLIMPORT int default_statistics_target; /* PGDLLIMPORT for PostGIS */ extern PGDLLIMPORT int vacuum_freeze_min_age; @@ -386,4 +407,9 @@ extern double anl_random_fract(void); extern double anl_init_selection_state(int n); extern double anl_get_next_S(double t, int n, double *stateptr); +extern void extvac_stats_start_idx(Relation rel, IndexBulkDeleteResult *stats, + LVExtStatCountersIdx *counters, BufferUsage *buffusage, WalUsage *walusage); +extern void extvac_stats_end_idx(Relation rel, IndexBulkDeleteResult *stats, + LVExtStatCountersIdx *counters, ExtVacReport *report, + BufferUsage *buffusage, WalUsage *walusage); #endif /* VACUUM_H */ diff --git a/src/include/pgstat.h b/src/include/pgstat.h index 818350af8d4..aef287ba81c 100644 --- a/src/include/pgstat.h +++ b/src/include/pgstat.h @@ -167,11 +167,19 @@ typedef struct PgStat_BackendSubEntry PgStat_Counter conflict_count[CONFLICT_NUM_TYPES]; } PgStat_BackendSubEntry; +/* Type of ExtVacReport */ +typedef enum ExtVacReportType +{ + PGSTAT_EXTVAC_INVALID = 0, + PGSTAT_EXTVAC_TABLE = 1, + PGSTAT_EXTVAC_INDEX = 2 +} ExtVacReportType; + /* ---------- * * ExtVacReport * - * Additional statistics of vacuum processing over a heap relation. + * Additional statistics of vacuum processing over a relation. * pages_removed is the amount by which the physically shrank, * if any (ie the change in its total size on disk) * pages_deleted refer to free space within the index file @@ -200,18 +208,44 @@ typedef struct ExtVacReport double delay_time; /* how long vacuum slept in vacuum delay point, in msec */ double total_time; /* total time of a vacuum operation, in msec */ - int64 pages_scanned; /* heap pages examined (not skipped by VM) */ - int64 pages_removed; /* heap pages removed by vacuum "truncation" */ - int64 vm_new_frozen_pages; /* pages marked in VM as frozen */ - int64 vm_new_visible_pages; /* pages marked in VM as all-visible */ - int64 vm_new_visible_frozen_pages; /* pages marked in VM as all-visible and frozen */ - int64 missed_dead_tuples; /* tuples not pruned by vacuum due to failure to get a cleanup lock */ - int64 missed_dead_pages; /* pages with missed dead tuples */ int64 tuples_deleted; /* tuples deleted by vacuum */ - int64 tuples_frozen; /* tuples frozen up by vacuum */ - int64 recently_dead_tuples; /* deleted tuples that are still visible to some transaction */ - int64 index_vacuum_count; /* the number of index vacuumings */ - int64 wraparound_failsafe_count; /* the number of times to prevent workaround problem */ + + ExtVacReportType type; /* heap, index, etc. */ + + /* ---------- + * + * There are separate metrics of statistic for tables and indexes, + * which collect during vacuum. + * The union operator allows to combine these statistics + * so that each metric is assigned to a specific class of collected statistics. + * Such a combined structure was called per_type_stats. + * The name of the structure itself is not used anywhere, + * it exists only for understanding the code. + * ---------- + */ + union + { + struct + { + int64 pages_scanned; /* heap pages examined (not skipped by VM) */ + int64 pages_removed; /* heap pages removed by vacuum "truncation" */ + int64 pages_frozen; /* pages marked in VM as frozen */ + int64 pages_all_visible; /* pages marked in VM as all-visible */ + int64 tuples_frozen; /* tuples frozen up by vacuum */ + int64 recently_dead_tuples; /* deleted tuples that are still visible to some transaction */ + int64 vm_new_frozen_pages; /* pages marked in VM as frozen */ + int64 vm_new_visible_pages; /* pages marked in VM as all-visible */ + int64 vm_new_visible_frozen_pages; /* pages marked in VM as all-visible and frozen */ + int64 missed_dead_tuples; /* tuples not pruned by vacuum due to failure to get a cleanup lock */ + int64 missed_dead_pages; /* pages with missed dead tuples */ + int64 index_vacuum_count; /* number of index vacuumings */ + int64 wraparound_failsafe_count; /* the number of times to prevent workaround problem */ + } table; + struct + { + int64 pages_deleted; /* number of pages deleted by vacuum */ + } index; + } /* per_type_stats */; } ExtVacReport; /* ---------- @@ -734,7 +768,8 @@ extern PgStat_FunctionCounts *find_funcstat_entry(Oid func_id); extern void pgstat_create_relation(Relation rel); extern void pgstat_drop_relation(Relation rel); extern void pgstat_copy_relation_stats(Relation dst, Relation src); - +extern void pgstat_accumulate_extvac_stats(ExtVacReport *dst, ExtVacReport *src, + bool accumulate_reltype_specific_info); extern void pgstat_init_relation(Relation rel); extern void pgstat_assoc_relation(Relation rel); extern void pgstat_unlink_relation(Relation rel); diff --git a/src/include/utils/pgstat_internal.h b/src/include/utils/pgstat_internal.h index 877218723dc..4f836e7fca0 100644 --- a/src/include/utils/pgstat_internal.h +++ b/src/include/utils/pgstat_internal.h @@ -600,7 +600,6 @@ extern PgStat_EntryRef *pgstat_fetch_pending_entry(PgStat_Kind kind, extern void *pgstat_fetch_entry(PgStat_Kind kind, Oid dboid, uint64 objid); extern void pgstat_snapshot_fixed(PgStat_Kind kind); -extern void pgstat_update_snapshot(PgStat_Kind kind); /* * Functions in pgstat_archiver.c */ diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out index dd795d58dfc..2becc7f3885 100644 --- a/src/test/regress/expected/rules.out +++ b/src/test/regress/expected/rules.out @@ -2245,6 +2245,28 @@ pg_stat_user_tables| SELECT relid, rev_all_visible_pages FROM pg_stat_all_tables WHERE ((schemaname <> ALL (ARRAY['pg_catalog'::name, 'information_schema'::name])) AND (schemaname !~ '^pg_toast'::text)); +pg_stat_vacuum_indexes| SELECT rel.oid AS relid, + ns.nspname AS schema, + rel.relname, + stats.total_blks_read, + stats.total_blks_hit, + stats.total_blks_dirtied, + stats.total_blks_written, + stats.rel_blks_read, + stats.rel_blks_hit, + stats.pages_deleted, + stats.tuples_deleted, + stats.wal_records, + stats.wal_fpi, + stats.wal_bytes, + stats.blk_read_time, + stats.blk_write_time, + stats.delay_time, + stats.total_time + FROM (pg_class rel + JOIN pg_namespace ns ON ((ns.oid = rel.relnamespace))), + LATERAL pg_stat_get_vacuum_indexes(rel.oid) stats(relid, total_blks_read, total_blks_hit, total_blks_dirtied, total_blks_written, rel_blks_read, rel_blks_hit, pages_deleted, tuples_deleted, wal_records, wal_fpi, wal_bytes, blk_read_time, blk_write_time, delay_time, total_time) + WHERE (rel.relkind = 'i'::"char"); pg_stat_vacuum_tables| SELECT ns.nspname AS schema, rel.relname, stats.relid, diff --git a/src/test/regress/expected/vacuum_index_statistics.out b/src/test/regress/expected/vacuum_index_statistics.out new file mode 100644 index 00000000000..b840a6ed4fe --- /dev/null +++ b/src/test/regress/expected/vacuum_index_statistics.out @@ -0,0 +1,183 @@ +-- +-- Test cumulative vacuum stats system +-- +-- Check the wall statistics collected during vacuum operation: +-- number of frozen and visible pages set by vacuum; +-- number of frozen and visible pages removed by backend. +-- Statistic wal_fpi is not displayed in this test because its behavior is unstable. +-- +-- conditio sine qua non +SHOW track_counts; -- must be on + track_counts +-------------- + on +(1 row) + +\set sample_size 10000 +-- not enabled by default, but we want to test it... +SET track_functions TO 'all'; +-- Test that vacuum statistics will be empty when parameter is off. +SET track_vacuum_statistics TO 'off'; +CREATE TABLE vestat (x int) WITH (autovacuum_enabled = off, fillfactor = 10); +INSERT INTO vestat SELECT x FROM generate_series(1,:sample_size) as x; +ANALYZE vestat; +DELETE FROM vestat WHERE x % 2 = 0; +VACUUM (PARALLEL 0, BUFFER_USAGE_LIMIT 128) vestat; +-- Must be empty. +SELECT * +FROM pg_stat_vacuum_indexes vt +WHERE vt.relname = 'vestat'; + relid | schema | relname | total_blks_read | total_blks_hit | total_blks_dirtied | total_blks_written | rel_blks_read | rel_blks_hit | pages_deleted | tuples_deleted | wal_records | wal_fpi | wal_bytes | blk_read_time | blk_write_time | delay_time | total_time +-------+--------+---------+-----------------+----------------+--------------------+--------------------+---------------+--------------+---------------+----------------+-------------+---------+-----------+---------------+----------------+------------+------------ +(0 rows) + +RESET track_vacuum_statistics; +DROP TABLE vestat CASCADE; +SHOW track_vacuum_statistics; -- must be on + track_vacuum_statistics +------------------------- + on +(1 row) + +-- ensure pending stats are flushed +SELECT pg_stat_force_next_flush(); + pg_stat_force_next_flush +-------------------------- + +(1 row) + +\set sample_size 10000 +SET vacuum_freeze_min_age = 0; +SET vacuum_freeze_table_age = 0; +--SET stats_fetch_consistency = snapshot; +CREATE TABLE vestat (x int primary key) WITH (autovacuum_enabled = off, fillfactor = 10); +INSERT INTO vestat SELECT x FROM generate_series(1,:sample_size) as x; +ANALYZE vestat; +SELECT oid AS ioid from pg_class where relname = 'vestat_pkey' \gset +DELETE FROM vestat WHERE x % 2 = 0; +-- Before the first vacuum execution extended stats view is empty. +SELECT vt.relname,relpages,pages_deleted,tuples_deleted +FROM pg_stat_vacuum_indexes vt, pg_class c +WHERE vt.relname = 'vestat_pkey' AND vt.relid = c.oid; + relname | relpages | pages_deleted | tuples_deleted +-------------+----------+---------------+---------------- + vestat_pkey | 30 | 0 | 0 +(1 row) + +SELECT relpages AS irp +FROM pg_class c +WHERE relname = 'vestat_pkey' \gset +VACUUM (PARALLEL 0, BUFFER_USAGE_LIMIT 128, INDEX_CLEANUP ON) vestat; +-- it is necessary to check the wal statistics +CHECKPOINT; +-- The table and index extended vacuum statistics should show us that +-- vacuum frozed pages and clean up pages, but pages_removed stayed the same +-- because of not full table have cleaned up +SELECT vt.relname,relpages-:irp = 0 AS relpages,pages_deleted = 0 AS pages_deleted,tuples_deleted > 0 AS tuples_deleted +FROM pg_stat_vacuum_indexes vt, pg_class c +WHERE vt.relname = 'vestat_pkey' AND vt.relid = c.oid; + relname | relpages | pages_deleted | tuples_deleted +-------------+----------+---------------+---------------- + vestat_pkey | t | t | t +(1 row) + +SELECT vt.relname,relpages AS irp,pages_deleted AS ipd,tuples_deleted AS itd +FROM pg_stat_vacuum_indexes vt, pg_class c +WHERE vt.relname = 'vestat_pkey' AND vt.relid = c.oid \gset +-- Store WAL advances into variables +SELECT wal_records AS iwr,wal_bytes AS iwb,wal_fpi AS ifpi FROM pg_stat_vacuum_indexes WHERE relname = 'vestat_pkey' \gset +-- Look into WAL records deltas. +SELECT wal_records > 0 AS diWR, wal_bytes > 0 AS diWB +FROM pg_stat_vacuum_indexes WHERE relname = 'vestat_pkey'; + diwr | diwb +------+------ + t | t +(1 row) + +DELETE FROM vestat;; +VACUUM (PARALLEL 0, BUFFER_USAGE_LIMIT 128, INDEX_CLEANUP ON) vestat; +-- it is necessary to check the wal statistics +CHECKPOINT; +-- pages_removed must be increased +SELECT vt.relname,relpages-:irp = 0 AS relpages,pages_deleted-:ipd > 0 AS pages_deleted,tuples_deleted-:itd > 0 AS tuples_deleted +FROM pg_stat_vacuum_indexes vt, pg_class c +WHERE vt.relname = 'vestat_pkey' AND vt.relid = c.oid; + relname | relpages | pages_deleted | tuples_deleted +-------------+----------+---------------+---------------- + vestat_pkey | t | t | t +(1 row) + +SELECT vt.relname,relpages AS irp,pages_deleted AS ipd,tuples_deleted AS itd +FROM pg_stat_vacuum_indexes vt, pg_class c +WHERE vt.relname = 'vestat_pkey' AND vt.relid = c.oid \gset +-- Store WAL advances into variables +SELECT wal_records-:iwr AS diwr, wal_bytes-:iwb AS diwb, wal_fpi-:ifpi AS difpi +FROM pg_stat_vacuum_indexes WHERE relname = 'vestat_pkey' \gset +-- WAL advance should be detected. +SELECT :diwr > 0 AS diWR, :diwb > 0 AS diWB; + diwr | diwb +------+------ + t | t +(1 row) + +-- Store WAL advances into variables +SELECT wal_records AS iwr,wal_bytes AS iwb,wal_fpi AS ifpi FROM pg_stat_vacuum_indexes WHERE relname = 'vestat_pkey' \gset +INSERT INTO vestat SELECT x FROM generate_series(1,:sample_size) as x; +DELETE FROM vestat WHERE x % 2 = 0; +-- VACUUM FULL doesn't report to stat collector. So, no any advancements of statistics +-- are detected here. +VACUUM FULL vestat; +-- It is necessary to check the wal statistics +CHECKPOINT; +-- Store WAL advances into variables +SELECT wal_records-:iwr AS diwr2, wal_bytes-:iwb AS diwb2, wal_fpi-:ifpi AS difpi2 +FROM pg_stat_vacuum_indexes WHERE relname = 'vestat_pkey' \gset +-- WAL and other statistics advance should not be detected. +SELECT :diwr2=0 AS diWR, :difpi2=0 AS iFPI, :diwb2=0 AS diWB; + diwr | ifpi | diwb +------+------+------ + t | t | t +(1 row) + +SELECT vt.relname,relpages-:irp < 0 AS relpages,pages_deleted-:ipd = 0 AS pages_deleted,tuples_deleted-:itd = 0 AS tuples_deleted +FROM pg_stat_vacuum_indexes vt, pg_class c +WHERE vt.relname = 'vestat_pkey' AND vt.relid = c.oid; + relname | relpages | pages_deleted | tuples_deleted +-------------+----------+---------------+---------------- + vestat_pkey | t | t | t +(1 row) + +SELECT vt.relname,relpages AS irp,pages_deleted AS ipd,tuples_deleted AS itd +FROM pg_stat_vacuum_indexes vt, pg_class c +WHERE vt.relname = 'vestat_pkey' AND vt.relid = c.oid \gset +-- Store WAL advances into variables +SELECT wal_records AS iwr,wal_bytes AS iwb,wal_fpi AS ifpi FROM pg_stat_vacuum_indexes WHERE relname = 'vestat_pkey' \gset +DELETE FROM vestat; +TRUNCATE vestat; +VACUUM (PARALLEL 0, BUFFER_USAGE_LIMIT 128, INDEX_CLEANUP ON) vestat; +-- it is necessary to check the wal statistics +CHECKPOINT; +-- Store WAL advances into variables after removing all tuples from the table +SELECT wal_records-:iwr AS diwr3, wal_bytes-:iwb AS diwb3, wal_fpi-:ifpi AS difpi3 +FROM pg_stat_vacuum_indexes WHERE relname = 'vestat_pkey' \gset +--There are nothing changed +SELECT :diwr3=0 AS diWR, :difpi3=0 AS iFPI, :diwb3=0 AS diWB; + diwr | ifpi | diwb +------+------+------ + t | t | t +(1 row) + +-- +-- Now, the table and index is compressed into zero number of pages. Check it +-- in vacuum extended statistics. +-- The pages_frozen, pages_scanned values shouldn't be changed +-- +SELECT vt.relname,relpages-:irp = 0 AS relpages,pages_deleted-:ipd = 0 AS pages_deleted,tuples_deleted-:itd = 0 AS tuples_deleted +FROM pg_stat_vacuum_indexes vt, pg_class c +WHERE vt.relname = 'vestat_pkey' AND vt.relid = c.oid; + relname | relpages | pages_deleted | tuples_deleted +-------------+----------+---------------+---------------- + vestat_pkey | f | t | t +(1 row) + +DROP TABLE vestat; diff --git a/src/test/regress/expected/vacuum_tables_statistics.out b/src/test/regress/expected/vacuum_tables_statistics.out index 0c05a812dd1..119c7abea5f 100644 --- a/src/test/regress/expected/vacuum_tables_statistics.out +++ b/src/test/regress/expected/vacuum_tables_statistics.out @@ -181,17 +181,39 @@ WHERE vt.relname = 'vestat' AND vt.relid = c.oid; vestat | t | t | f | t | t (1 row) +DROP TABLE vestat; +SELECT pg_stat_force_next_flush(); + pg_stat_force_next_flush +-------------------------- + +(1 row) + +SET vacuum_freeze_min_age = 0; +SET vacuum_freeze_table_age = 0; +CREATE TABLE vestat (x int) WITH (autovacuum_enabled = off, fillfactor = 10); INSERT INTO vestat SELECT x FROM generate_series(1,:sample_size) as x; ANALYZE vestat; +SELECT pg_stat_force_next_flush(); + pg_stat_force_next_flush +-------------------------- + +(1 row) + -- must be empty SELECT vm_new_frozen_pages, vm_new_visible_pages, rev_all_frozen_pages,rev_all_visible_pages,vm_new_visible_frozen_pages FROM pg_stat_vacuum_tables, pg_stat_all_tables WHERE pg_stat_vacuum_tables.relname = 'vestat' and pg_stat_vacuum_tables.relid = pg_stat_all_tables.relid; vm_new_frozen_pages | vm_new_visible_pages | rev_all_frozen_pages | rev_all_visible_pages | vm_new_visible_frozen_pages ---------------------+----------------------+----------------------+-----------------------+----------------------------- - 0 | 910 | 0 | 0 | 455 + 0 | 0 | 0 | 0 | 0 (1 row) VACUUM (PARALLEL 0, BUFFER_USAGE_LIMIT 128) vestat; +SELECT pg_stat_force_next_flush(); + pg_stat_force_next_flush +-------------------------- + +(1 row) + -- backend defreezed pages SELECT vm_new_frozen_pages > 0 AS vm_new_frozen_pages,vm_new_visible_pages > 0 AS vm_new_visible_pages,vm_new_visible_frozen_pages > 0 AS vm_new_visible_frozen_pages,rev_all_frozen_pages = 0 AS rev_all_frozen_pages,rev_all_visible_pages = 0 AS rev_all_visible_pages FROM pg_stat_vacuum_tables, pg_stat_all_tables WHERE pg_stat_vacuum_tables.relname = 'vestat' and pg_stat_vacuum_tables.relid = pg_stat_all_tables.relid; @@ -204,16 +226,28 @@ SELECT vm_new_frozen_pages AS pf, vm_new_visible_pages AS pv,vm_new_visible_froz FROM pg_stat_vacuum_tables, pg_stat_all_tables WHERE pg_stat_vacuum_tables.relname = 'vestat' and pg_stat_vacuum_tables.relid = pg_stat_all_tables.relid \gset UPDATE vestat SET x = x + 1001; VACUUM (PARALLEL 0, BUFFER_USAGE_LIMIT 128) vestat; +SELECT pg_stat_force_next_flush(); + pg_stat_force_next_flush +-------------------------- + +(1 row) + SELECT vm_new_frozen_pages > :pf AS vm_new_frozen_pages,vm_new_visible_pages > :pv AS vm_new_visible_pages,vm_new_visible_frozen_pages > :pvf AS vm_new_visible_frozen_pages,rev_all_frozen_pages > :hafp AS rev_all_frozen_pages,rev_all_visible_pages > :havp AS rev_all_visible_pages FROM pg_stat_vacuum_tables, pg_stat_all_tables WHERE pg_stat_vacuum_tables.relname = 'vestat' and pg_stat_vacuum_tables.relid = pg_stat_all_tables.relid; vm_new_frozen_pages | vm_new_visible_pages | vm_new_visible_frozen_pages | rev_all_frozen_pages | rev_all_visible_pages ---------------------+----------------------+-----------------------------+----------------------+----------------------- - f | t | f | f | f + f | t | t | t | t (1 row) SELECT vm_new_frozen_pages AS pf, vm_new_visible_pages AS pv, vm_new_visible_frozen_pages AS pvf, rev_all_frozen_pages AS hafp,rev_all_visible_pages AS havp FROM pg_stat_vacuum_tables, pg_stat_all_tables WHERE pg_stat_vacuum_tables.relname = 'vestat' and pg_stat_vacuum_tables.relid = pg_stat_all_tables.relid \gset VACUUM (PARALLEL 0, BUFFER_USAGE_LIMIT 128) vestat; +SELECT pg_stat_force_next_flush(); + pg_stat_force_next_flush +-------------------------- + +(1 row) + -- vacuum freezed pages SELECT vm_new_frozen_pages = :pf AS vm_new_frozen_pages,vm_new_visible_pages = :pv AS vm_new_visible_pages,vm_new_visible_frozen_pages = :pvf AS vm_new_visible_frozen_pages, rev_all_frozen_pages = :hafp AS rev_all_frozen_pages,rev_all_visible_pages = :havp AS rev_all_visible_pages FROM pg_stat_vacuum_tables, pg_stat_all_tables WHERE pg_stat_vacuum_tables.relname = 'vestat' and pg_stat_vacuum_tables.relid = pg_stat_all_tables.relid; @@ -222,4 +256,6 @@ FROM pg_stat_vacuum_tables, pg_stat_all_tables WHERE pg_stat_vacuum_tables.relna t | t | t | t | t (1 row) +RESET vacuum_freeze_min_age; +RESET vacuum_freeze_table_age; DROP TABLE vestat CASCADE; diff --git a/src/test/regress/parallel_schedule b/src/test/regress/parallel_schedule index d11f6b7ef4b..977a87a5b1f 100644 --- a/src/test/regress/parallel_schedule +++ b/src/test/regress/parallel_schedule @@ -140,4 +140,5 @@ test: tablespace # ---------- # Check vacuum statistics # ---------- +test: vacuum_index_statistics test: vacuum_tables_statistics \ No newline at end of file diff --git a/src/test/regress/sql/vacuum_index_statistics.sql b/src/test/regress/sql/vacuum_index_statistics.sql new file mode 100644 index 00000000000..ae146e1d23f --- /dev/null +++ b/src/test/regress/sql/vacuum_index_statistics.sql @@ -0,0 +1,151 @@ +-- +-- Test cumulative vacuum stats system +-- +-- Check the wall statistics collected during vacuum operation: +-- number of frozen and visible pages set by vacuum; +-- number of frozen and visible pages removed by backend. +-- Statistic wal_fpi is not displayed in this test because its behavior is unstable. +-- +-- conditio sine qua non +SHOW track_counts; -- must be on + +\set sample_size 10000 + +-- not enabled by default, but we want to test it... +SET track_functions TO 'all'; + +-- Test that vacuum statistics will be empty when parameter is off. +SET track_vacuum_statistics TO 'off'; + +CREATE TABLE vestat (x int) WITH (autovacuum_enabled = off, fillfactor = 10); +INSERT INTO vestat SELECT x FROM generate_series(1,:sample_size) as x; +ANALYZE vestat; + +DELETE FROM vestat WHERE x % 2 = 0; + +VACUUM (PARALLEL 0, BUFFER_USAGE_LIMIT 128) vestat; + +-- Must be empty. +SELECT * +FROM pg_stat_vacuum_indexes vt +WHERE vt.relname = 'vestat'; + +RESET track_vacuum_statistics; +DROP TABLE vestat CASCADE; + +SHOW track_vacuum_statistics; -- must be on + +-- ensure pending stats are flushed +SELECT pg_stat_force_next_flush(); + +\set sample_size 10000 +SET vacuum_freeze_min_age = 0; +SET vacuum_freeze_table_age = 0; +--SET stats_fetch_consistency = snapshot; +CREATE TABLE vestat (x int primary key) WITH (autovacuum_enabled = off, fillfactor = 10); +INSERT INTO vestat SELECT x FROM generate_series(1,:sample_size) as x; +ANALYZE vestat; + +SELECT oid AS ioid from pg_class where relname = 'vestat_pkey' \gset + +DELETE FROM vestat WHERE x % 2 = 0; +-- Before the first vacuum execution extended stats view is empty. +SELECT vt.relname,relpages,pages_deleted,tuples_deleted +FROM pg_stat_vacuum_indexes vt, pg_class c +WHERE vt.relname = 'vestat_pkey' AND vt.relid = c.oid; +SELECT relpages AS irp +FROM pg_class c +WHERE relname = 'vestat_pkey' \gset + +VACUUM (PARALLEL 0, BUFFER_USAGE_LIMIT 128, INDEX_CLEANUP ON) vestat; +-- it is necessary to check the wal statistics +CHECKPOINT; + +-- The table and index extended vacuum statistics should show us that +-- vacuum frozed pages and clean up pages, but pages_removed stayed the same +-- because of not full table have cleaned up +SELECT vt.relname,relpages-:irp = 0 AS relpages,pages_deleted = 0 AS pages_deleted,tuples_deleted > 0 AS tuples_deleted +FROM pg_stat_vacuum_indexes vt, pg_class c +WHERE vt.relname = 'vestat_pkey' AND vt.relid = c.oid; +SELECT vt.relname,relpages AS irp,pages_deleted AS ipd,tuples_deleted AS itd +FROM pg_stat_vacuum_indexes vt, pg_class c +WHERE vt.relname = 'vestat_pkey' AND vt.relid = c.oid \gset + +-- Store WAL advances into variables +SELECT wal_records AS iwr,wal_bytes AS iwb,wal_fpi AS ifpi FROM pg_stat_vacuum_indexes WHERE relname = 'vestat_pkey' \gset + +-- Look into WAL records deltas. +SELECT wal_records > 0 AS diWR, wal_bytes > 0 AS diWB +FROM pg_stat_vacuum_indexes WHERE relname = 'vestat_pkey'; + +DELETE FROM vestat;; +VACUUM (PARALLEL 0, BUFFER_USAGE_LIMIT 128, INDEX_CLEANUP ON) vestat; +-- it is necessary to check the wal statistics +CHECKPOINT; + +-- pages_removed must be increased +SELECT vt.relname,relpages-:irp = 0 AS relpages,pages_deleted-:ipd > 0 AS pages_deleted,tuples_deleted-:itd > 0 AS tuples_deleted +FROM pg_stat_vacuum_indexes vt, pg_class c +WHERE vt.relname = 'vestat_pkey' AND vt.relid = c.oid; +SELECT vt.relname,relpages AS irp,pages_deleted AS ipd,tuples_deleted AS itd +FROM pg_stat_vacuum_indexes vt, pg_class c +WHERE vt.relname = 'vestat_pkey' AND vt.relid = c.oid \gset + +-- Store WAL advances into variables +SELECT wal_records-:iwr AS diwr, wal_bytes-:iwb AS diwb, wal_fpi-:ifpi AS difpi +FROM pg_stat_vacuum_indexes WHERE relname = 'vestat_pkey' \gset + +-- WAL advance should be detected. +SELECT :diwr > 0 AS diWR, :diwb > 0 AS diWB; + +-- Store WAL advances into variables +SELECT wal_records AS iwr,wal_bytes AS iwb,wal_fpi AS ifpi FROM pg_stat_vacuum_indexes WHERE relname = 'vestat_pkey' \gset + +INSERT INTO vestat SELECT x FROM generate_series(1,:sample_size) as x; +DELETE FROM vestat WHERE x % 2 = 0; +-- VACUUM FULL doesn't report to stat collector. So, no any advancements of statistics +-- are detected here. +VACUUM FULL vestat; +-- It is necessary to check the wal statistics +CHECKPOINT; + +-- Store WAL advances into variables +SELECT wal_records-:iwr AS diwr2, wal_bytes-:iwb AS diwb2, wal_fpi-:ifpi AS difpi2 +FROM pg_stat_vacuum_indexes WHERE relname = 'vestat_pkey' \gset + +-- WAL and other statistics advance should not be detected. +SELECT :diwr2=0 AS diWR, :difpi2=0 AS iFPI, :diwb2=0 AS diWB; + +SELECT vt.relname,relpages-:irp < 0 AS relpages,pages_deleted-:ipd = 0 AS pages_deleted,tuples_deleted-:itd = 0 AS tuples_deleted +FROM pg_stat_vacuum_indexes vt, pg_class c +WHERE vt.relname = 'vestat_pkey' AND vt.relid = c.oid; +SELECT vt.relname,relpages AS irp,pages_deleted AS ipd,tuples_deleted AS itd +FROM pg_stat_vacuum_indexes vt, pg_class c +WHERE vt.relname = 'vestat_pkey' AND vt.relid = c.oid \gset + +-- Store WAL advances into variables +SELECT wal_records AS iwr,wal_bytes AS iwb,wal_fpi AS ifpi FROM pg_stat_vacuum_indexes WHERE relname = 'vestat_pkey' \gset + +DELETE FROM vestat; +TRUNCATE vestat; +VACUUM (PARALLEL 0, BUFFER_USAGE_LIMIT 128, INDEX_CLEANUP ON) vestat; +-- it is necessary to check the wal statistics +CHECKPOINT; + +-- Store WAL advances into variables after removing all tuples from the table +SELECT wal_records-:iwr AS diwr3, wal_bytes-:iwb AS diwb3, wal_fpi-:ifpi AS difpi3 +FROM pg_stat_vacuum_indexes WHERE relname = 'vestat_pkey' \gset + +--There are nothing changed +SELECT :diwr3=0 AS diWR, :difpi3=0 AS iFPI, :diwb3=0 AS diWB; + +-- +-- Now, the table and index is compressed into zero number of pages. Check it +-- in vacuum extended statistics. +-- The pages_frozen, pages_scanned values shouldn't be changed +-- +SELECT vt.relname,relpages-:irp = 0 AS relpages,pages_deleted-:ipd = 0 AS pages_deleted,tuples_deleted-:itd = 0 AS tuples_deleted +FROM pg_stat_vacuum_indexes vt, pg_class c +WHERE vt.relname = 'vestat_pkey' AND vt.relid = c.oid; + +DROP TABLE vestat; diff --git a/src/test/regress/sql/vacuum_tables_statistics.sql b/src/test/regress/sql/vacuum_tables_statistics.sql index 8ad69108ca1..dfd7af70027 100644 --- a/src/test/regress/sql/vacuum_tables_statistics.sql +++ b/src/test/regress/sql/vacuum_tables_statistics.sql @@ -148,14 +148,22 @@ SELECT vt.relname,vm_new_frozen_pages-:fp = 0 AS vm_new_frozen_pages,tuples_dele FROM pg_stat_vacuum_tables vt, pg_class c WHERE vt.relname = 'vestat' AND vt.relid = c.oid; +DROP TABLE vestat; +SELECT pg_stat_force_next_flush(); +SET vacuum_freeze_min_age = 0; +SET vacuum_freeze_table_age = 0; +CREATE TABLE vestat (x int) WITH (autovacuum_enabled = off, fillfactor = 10); + INSERT INTO vestat SELECT x FROM generate_series(1,:sample_size) as x; ANALYZE vestat; +SELECT pg_stat_force_next_flush(); -- must be empty SELECT vm_new_frozen_pages, vm_new_visible_pages, rev_all_frozen_pages,rev_all_visible_pages,vm_new_visible_frozen_pages FROM pg_stat_vacuum_tables, pg_stat_all_tables WHERE pg_stat_vacuum_tables.relname = 'vestat' and pg_stat_vacuum_tables.relid = pg_stat_all_tables.relid; VACUUM (PARALLEL 0, BUFFER_USAGE_LIMIT 128) vestat; +SELECT pg_stat_force_next_flush(); -- backend defreezed pages SELECT vm_new_frozen_pages > 0 AS vm_new_frozen_pages,vm_new_visible_pages > 0 AS vm_new_visible_pages,vm_new_visible_frozen_pages > 0 AS vm_new_visible_frozen_pages,rev_all_frozen_pages = 0 AS rev_all_frozen_pages,rev_all_visible_pages = 0 AS rev_all_visible_pages @@ -165,6 +173,7 @@ FROM pg_stat_vacuum_tables, pg_stat_all_tables WHERE pg_stat_vacuum_tables.relna UPDATE vestat SET x = x + 1001; VACUUM (PARALLEL 0, BUFFER_USAGE_LIMIT 128) vestat; +SELECT pg_stat_force_next_flush(); SELECT vm_new_frozen_pages > :pf AS vm_new_frozen_pages,vm_new_visible_pages > :pv AS vm_new_visible_pages,vm_new_visible_frozen_pages > :pvf AS vm_new_visible_frozen_pages,rev_all_frozen_pages > :hafp AS rev_all_frozen_pages,rev_all_visible_pages > :havp AS rev_all_visible_pages FROM pg_stat_vacuum_tables, pg_stat_all_tables WHERE pg_stat_vacuum_tables.relname = 'vestat' and pg_stat_vacuum_tables.relid = pg_stat_all_tables.relid; @@ -172,9 +181,12 @@ SELECT vm_new_frozen_pages AS pf, vm_new_visible_pages AS pv, vm_new_visible_fro FROM pg_stat_vacuum_tables, pg_stat_all_tables WHERE pg_stat_vacuum_tables.relname = 'vestat' and pg_stat_vacuum_tables.relid = pg_stat_all_tables.relid \gset VACUUM (PARALLEL 0, BUFFER_USAGE_LIMIT 128) vestat; +SELECT pg_stat_force_next_flush(); -- vacuum freezed pages SELECT vm_new_frozen_pages = :pf AS vm_new_frozen_pages,vm_new_visible_pages = :pv AS vm_new_visible_pages,vm_new_visible_frozen_pages = :pvf AS vm_new_visible_frozen_pages, rev_all_frozen_pages = :hafp AS rev_all_frozen_pages,rev_all_visible_pages = :havp AS rev_all_visible_pages FROM pg_stat_vacuum_tables, pg_stat_all_tables WHERE pg_stat_vacuum_tables.relname = 'vestat' and pg_stat_vacuum_tables.relid = pg_stat_all_tables.relid; +RESET vacuum_freeze_min_age; +RESET vacuum_freeze_table_age; DROP TABLE vestat CASCADE; -- 2.34.1