From fb8a110aa416e3314dabbf316df1897c5e244008 Mon Sep 17 00:00:00 2001 From: Alena Rybakina Date: Thu, 12 Dec 2024 11:17:01 +0300 Subject: [PATCH 1/4] Machinery for grabbing an extended vacuum statistics on heap relations. Value of total_blks_hit, total_blks_read, total_blks_dirtied are number of hitted, missed and dirtied pages in shared buffers during a vacuum operation respectively. total_blks_dirtied means 'dirtied only by this action'. So, if this page was dirty before the vacuum operation, it doesn't count this page as 'dirtied'. The tuples_deleted parameter is the number of tuples cleaned up by the vacuum operation. The delay_time value means total vacuum sleep time in vacuum delay point. The pages_removed value is the number of pages by which the physical data storage of the relation was reduced. The value of pages_deleted parameter is the number of freed pages in the table (file size may not have changed). Interruptions number of (auto)vacuum process during vacuuming of a relation. We report from the vacuum_error_callback routine. So we can log all ERROR reports. In the case of autovacuum we can report SIGINT signals too. It maybe dangerous to make such complex task (send) in an error callback - we can catch ERROR in ERROR problem. But it looks like we have so small chance to stuck into this problem. So, let's try to use. This parameter relates to a problem, covered by b19e4250. Tracking of IO during an (auto)vacuum operation. Introduced variables blk_read_time and blk_write_time tracks only access to buffer pages and flushing them to disk. Reading operation is trivial, but writing measurement technique is not obvious. So, during a vacuum writing time can be zero incremented because no any flushing operations were performed. System time and user time are parameters that describes how much time a vacuum operation has spent in executing of code in user space and kernel space accordingly. Also, accumulate total time of a vacuum that is a diff between timestamps in start and finish points in the vacuum code. Remember about idle time, when vacuum waited for IO and locks, so total time isn't equal a sum of user and system time, but no less. pages_frozen - number of pages that are marked as frozen in vm during vacuum. This parameter is incremented if page is marked as all-frozen. pages_all_visible - number of pages that are marked as all-visible in vm during vacuum. Authors: Alena Rybakina , Andrei Lepikhov , Andrei Zubkov Reviewed-by: Dilip Kumar , Masahiko Sawada , Ilia Evdokimov , jian he , Kirill Reshke , Alexander Korotkov , Jim Nasby --- src/backend/access/heap/vacuumlazy.c | 146 +++++++++++- src/backend/access/heap/visibilitymap.c | 10 + src/backend/catalog/system_views.sql | 51 +++- src/backend/commands/vacuum.c | 4 + src/backend/commands/vacuumparallel.c | 1 + src/backend/utils/activity/pgstat.c | 12 +- src/backend/utils/activity/pgstat_relation.c | 46 +++- src/backend/utils/adt/pgstatfuncs.c | 144 +++++++++++ src/backend/utils/error/elog.c | 13 + src/backend/utils/misc/guc_tables.c | 9 + src/include/catalog/pg_proc.dat | 18 ++ src/include/commands/vacuum.h | 1 + src/include/pgstat.h | 81 ++++++- src/include/utils/elog.h | 1 + src/include/utils/pgstat_internal.h | 2 +- .../vacuum-extending-in-repetable-read.out | 53 +++++ src/test/isolation/isolation_schedule | 1 + .../vacuum-extending-in-repetable-read.spec | 53 +++++ src/test/regress/expected/rules.out | 43 +++- .../expected/vacuum_tables_statistics.out | 225 ++++++++++++++++++ src/test/regress/parallel_schedule | 5 + .../regress/sql/vacuum_tables_statistics.sql | 180 ++++++++++++++ 22 files changed, 1081 insertions(+), 18 deletions(-) create mode 100644 src/test/isolation/expected/vacuum-extending-in-repetable-read.out create mode 100644 src/test/isolation/specs/vacuum-extending-in-repetable-read.spec create mode 100644 src/test/regress/expected/vacuum_tables_statistics.out create mode 100644 src/test/regress/sql/vacuum_tables_statistics.sql diff --git a/src/backend/access/heap/vacuumlazy.c b/src/backend/access/heap/vacuumlazy.c index f2ca9430581..987bd529dec 100644 --- a/src/backend/access/heap/vacuumlazy.c +++ b/src/backend/access/heap/vacuumlazy.c @@ -165,6 +165,7 @@ typedef struct LVRelState /* Error reporting state */ char *dbname; char *relnamespace; + Oid reloid; char *relname; char *indname; /* Current index name */ BlockNumber blkno; /* used only for heap operations */ @@ -239,6 +240,18 @@ typedef struct LVSavedErrInfo VacErrPhase phase; } LVSavedErrInfo; +/* + * Counters and usage data for extended stats tracking. + */ +typedef struct LVExtStatCounters +{ + TimestampTz starttime; + WalUsage walusage; + BufferUsage bufusage; + double VacuumDelayTime; + PgStat_Counter blocks_fetched; + PgStat_Counter blocks_hit; +} LVExtStatCounters; /* non-export function prototypes */ static void lazy_scan_heap(LVRelState *vacrel); @@ -292,6 +305,106 @@ static void update_vacuum_error_info(LVRelState *vacrel, static void restore_vacuum_error_info(LVRelState *vacrel, const LVSavedErrInfo *saved_vacrel); +/* ---------- + * extvac_stats_start() - + * + * Save cut-off values of extended vacuum counters before start of a relation + * processing. + * ---------- + */ +static void +extvac_stats_start(Relation rel, LVExtStatCounters *counters) +{ + TimestampTz starttime; + + if(!pgstat_track_vacuum_statistics) + return; + + memset(counters, 0, sizeof(LVExtStatCounters)); + + starttime = GetCurrentTimestamp(); + + counters->starttime = starttime; + counters->walusage = pgWalUsage; + counters->bufusage = pgBufferUsage; + counters->VacuumDelayTime = VacuumDelayTime; + counters->blocks_fetched = 0; + counters->blocks_hit = 0; + + if (!rel->pgstat_info || !pgstat_track_counts) + /* + * if something goes wrong or user doesn't want to track a database + * activity - just suppress it. + */ + return; + + counters->blocks_fetched = rel->pgstat_info->counts.blocks_fetched; + counters->blocks_hit = rel->pgstat_info->counts.blocks_hit; +} + +/* ---------- + * extvac_stats_end() - + * + * Called to finish an extended vacuum statistic gathering and form a report. + * ---------- + */ +static void +extvac_stats_end(Relation rel, LVExtStatCounters *counters, + ExtVacReport *report) +{ + WalUsage walusage; + BufferUsage bufusage; + TimestampTz endtime; + long secs; + int usecs; + + if(!pgstat_track_vacuum_statistics) + return; + + /* Calculate diffs of global stat parameters on WAL and buffer usage. */ + memset(&walusage, 0, sizeof(WalUsage)); + WalUsageAccumDiff(&walusage, &pgWalUsage, &counters->walusage); + + memset(&bufusage, 0, sizeof(BufferUsage)); + BufferUsageAccumDiff(&bufusage, &pgBufferUsage, &counters->bufusage); + + endtime = GetCurrentTimestamp(); + TimestampDifference(counters->starttime, endtime, &secs, &usecs); + + memset(report, 0, sizeof(ExtVacReport)); + + /* + * Fill additional statistics on a vacuum processing operation. + */ + report->total_blks_read = bufusage.local_blks_read + bufusage.shared_blks_read; + report->total_blks_hit = bufusage.local_blks_hit + bufusage.shared_blks_hit; + report->total_blks_dirtied = bufusage.local_blks_dirtied + bufusage.shared_blks_dirtied; + report->total_blks_written = bufusage.shared_blks_written; + + report->wal_records = walusage.wal_records; + report->wal_fpi = walusage.wal_fpi; + report->wal_bytes = walusage.wal_bytes; + + report->blk_read_time = INSTR_TIME_GET_MILLISEC(bufusage.local_blk_read_time); + report->blk_read_time += INSTR_TIME_GET_MILLISEC(bufusage.shared_blk_read_time); + report->blk_write_time = INSTR_TIME_GET_MILLISEC(bufusage.local_blk_write_time); + report->blk_write_time = INSTR_TIME_GET_MILLISEC(bufusage.shared_blk_write_time); + report->delay_time = VacuumDelayTime - counters->VacuumDelayTime; + + report->total_time = secs * 1000. + usecs / 1000.; + + if (!rel->pgstat_info || !pgstat_track_counts) + /* + * if something goes wrong or an user doesn't want to track a database + * activity - just suppress it. + */ + return; + + report->blks_fetched = + rel->pgstat_info->counts.blocks_fetched - counters->blocks_fetched; + report->blks_hit = + rel->pgstat_info->counts.blocks_hit - counters->blocks_hit; +} /* * heap_vacuum_rel() -- perform VACUUM for one heap relation @@ -324,7 +437,14 @@ heap_vacuum_rel(Relation rel, VacuumParams *params, WalUsage startwalusage = pgWalUsage; BufferUsage startbufferusage = pgBufferUsage; ErrorContextCallback errcallback; + LVExtStatCounters extVacCounters; + ExtVacReport extVacReport; char **indnames = NULL; + ExtVacReport allzero; + + /* Initialize vacuum statistics */ + memset(&allzero, 0, sizeof(ExtVacReport)); + extVacReport = allzero; verbose = (params->options & VACOPT_VERBOSE) != 0; instrument = (verbose || (AmAutoVacuumWorkerProcess() && @@ -342,7 +462,7 @@ heap_vacuum_rel(Relation rel, VacuumParams *params, pgstat_progress_start_command(PROGRESS_COMMAND_VACUUM, RelationGetRelid(rel)); - + extvac_stats_start(rel, &extVacCounters); /* * Setup error traceback support for ereport() first. The idea is to set * up an error context callback to display additional information on any @@ -359,6 +479,7 @@ heap_vacuum_rel(Relation rel, VacuumParams *params, vacrel->dbname = get_database_name(MyDatabaseId); vacrel->relnamespace = get_namespace_name(RelationGetNamespace(rel)); vacrel->relname = pstrdup(RelationGetRelationName(rel)); + vacrel->reloid = RelationGetRelid(rel); vacrel->indname = NULL; vacrel->phase = VACUUM_ERRCB_PHASE_UNKNOWN; vacrel->verbose = verbose; @@ -591,6 +712,26 @@ heap_vacuum_rel(Relation rel, VacuumParams *params, vacrel->NewRelfrozenXid, vacrel->NewRelminMxid, &frozenxid_updated, &minmulti_updated, false); + /* Make generic extended vacuum stats report */ + extvac_stats_end(rel, &extVacCounters, &extVacReport); + + if(pgstat_track_vacuum_statistics) + { + extVacReport.recently_dead_tuples = 0; + /* Fill heap-specific extended stats fields */ + extVacReport.pages_scanned = vacrel->scanned_pages; + extVacReport.pages_removed = vacrel->removed_pages; + extVacReport.vm_new_frozen_pages = vacrel->vm_new_frozen_pages; + extVacReport.vm_new_visible_pages = vacrel->vm_new_visible_pages; + extVacReport.vm_new_visible_frozen_pages = vacrel->vm_new_visible_frozen_pages; + extVacReport.tuples_deleted = vacrel->tuples_deleted; + extVacReport.tuples_frozen = vacrel->tuples_frozen; + extVacReport.recently_dead_tuples = vacrel->recently_dead_tuples; + extVacReport.missed_dead_tuples = vacrel->missed_dead_tuples; + extVacReport.missed_dead_pages = vacrel->missed_dead_pages; + extVacReport.index_vacuum_count = vacrel->num_index_scans; + } + /* * Report results to the cumulative stats system, too. * @@ -605,7 +746,8 @@ heap_vacuum_rel(Relation rel, VacuumParams *params, rel->rd_rel->relisshared, Max(vacrel->new_live_tuples, 0), vacrel->recently_dead_tuples + - vacrel->missed_dead_tuples); + vacrel->missed_dead_tuples, + &extVacReport); pgstat_progress_end_command(); if (instrument) diff --git a/src/backend/access/heap/visibilitymap.c b/src/backend/access/heap/visibilitymap.c index 79b79d5982e..ecfb0340328 100644 --- a/src/backend/access/heap/visibilitymap.c +++ b/src/backend/access/heap/visibilitymap.c @@ -91,6 +91,7 @@ #include "access/xloginsert.h" #include "access/xlogutils.h" #include "miscadmin.h" +#include "pgstat.h" #include "port/pg_bitutils.h" #include "storage/bufmgr.h" #include "storage/smgr.h" @@ -160,6 +161,15 @@ visibilitymap_clear(Relation rel, BlockNumber heapBlk, Buffer vmbuf, uint8 flags if (map[mapByte] & mask) { + /* + * As part of vacuum stats, track how often all-visible or all-frozen + * bits are cleared. + */ + if (map[mapByte] >> mapOffset & flags & VISIBILITYMAP_ALL_VISIBLE) + pgstat_count_vm_rev_all_visible(rel); + if (map[mapByte] >> mapOffset & flags & VISIBILITYMAP_ALL_FROZEN) + pgstat_count_vm_rev_all_frozen(rel); + map[mapByte] &= ~mask; MarkBufferDirty(vmbuf); diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index da9a8fe99f2..f3cb5b5bf90 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -691,7 +691,9 @@ CREATE VIEW pg_stat_all_tables AS pg_stat_get_vacuum_count(C.oid) AS vacuum_count, pg_stat_get_autovacuum_count(C.oid) AS autovacuum_count, pg_stat_get_analyze_count(C.oid) AS analyze_count, - pg_stat_get_autoanalyze_count(C.oid) AS autoanalyze_count + pg_stat_get_autoanalyze_count(C.oid) AS autoanalyze_count, + pg_stat_get_rev_all_frozen_pages(C.oid) AS rev_all_frozen_pages, + pg_stat_get_rev_all_visible_pages(C.oid) AS rev_all_visible_pages FROM pg_class C LEFT JOIN pg_index I ON C.oid = I.indrelid LEFT JOIN pg_namespace N ON (N.oid = C.relnamespace) @@ -1381,3 +1383,50 @@ CREATE VIEW pg_stat_subscription_stats AS CREATE VIEW pg_wait_events AS SELECT * FROM pg_get_wait_events(); +-- +-- Show extended cumulative statistics on a vacuum operation over all tables and +-- databases of the instance. +-- Use Invalid Oid "0" as an input relation id to get stat on each table in a +-- database. +-- + +CREATE VIEW pg_stat_vacuum_tables AS +SELECT + ns.nspname AS "schema", + rel.relname AS relname, + stats.relid as relid, + + stats.total_blks_read AS total_blks_read, + stats.total_blks_hit AS total_blks_hit, + stats.total_blks_dirtied AS total_blks_dirtied, + stats.total_blks_written AS total_blks_written, + + stats.rel_blks_read AS rel_blks_read, + stats.rel_blks_hit AS rel_blks_hit, + + stats.pages_scanned AS pages_scanned, + stats.pages_removed AS pages_removed, + stats.vm_new_frozen_pages AS vm_new_frozen_pages, + stats.vm_new_visible_pages AS vm_new_visible_pages, + stats.vm_new_visible_frozen_pages AS vm_new_visible_frozen_pages, + stats.missed_dead_pages AS missed_dead_pages, + stats.tuples_deleted AS tuples_deleted, + stats.tuples_frozen AS tuples_frozen, + stats.recently_dead_tuples AS recently_dead_tuples, + stats.missed_dead_tuples AS missed_dead_tuples, + + stats.index_vacuum_count AS index_vacuum_count, + stats.wal_records AS wal_records, + stats.wal_fpi AS wal_fpi, + stats.wal_bytes AS wal_bytes, + + stats.blk_read_time AS blk_read_time, + stats.blk_write_time AS blk_write_time, + + stats.delay_time AS delay_time, + stats.total_time AS total_time + +FROM pg_class rel + JOIN pg_namespace ns ON ns.oid = rel.relnamespace, + LATERAL pg_stat_get_vacuum_tables(rel.oid) stats +WHERE rel.relkind = 'r'; diff --git a/src/backend/commands/vacuum.c b/src/backend/commands/vacuum.c index bb639ef51fb..fa484dbe639 100644 --- a/src/backend/commands/vacuum.c +++ b/src/backend/commands/vacuum.c @@ -102,6 +102,9 @@ pg_atomic_uint32 *VacuumSharedCostBalance = NULL; pg_atomic_uint32 *VacuumActiveNWorkers = NULL; int VacuumCostBalanceLocal = 0; +/* Cumulative storage to report total vacuum delay time. */ +double VacuumDelayTime = 0; /* msec. */ + /* non-export function prototypes */ static List *expand_vacuum_rel(VacuumRelation *vrel, MemoryContext vac_context, int options); @@ -2419,6 +2422,7 @@ vacuum_delay_point(void) exit(1); VacuumCostBalance = 0; + VacuumDelayTime += msec; /* * Balance and update limit values for autovacuum workers. We must do diff --git a/src/backend/commands/vacuumparallel.c b/src/backend/commands/vacuumparallel.c index 67cba17a564..d7d9873c8ea 100644 --- a/src/backend/commands/vacuumparallel.c +++ b/src/backend/commands/vacuumparallel.c @@ -1047,6 +1047,7 @@ parallel_vacuum_main(dsm_segment *seg, shm_toc *toc) /* Set cost-based vacuum delay */ VacuumUpdateCosts(); VacuumCostBalance = 0; + VacuumDelayTime = 0; VacuumCostBalanceLocal = 0; VacuumSharedCostBalance = &(shared->cost_balance); VacuumActiveNWorkers = &(shared->active_nworkers); diff --git a/src/backend/utils/activity/pgstat.c b/src/backend/utils/activity/pgstat.c index b4e357c8a42..bee1461814b 100644 --- a/src/backend/utils/activity/pgstat.c +++ b/src/backend/utils/activity/pgstat.c @@ -190,7 +190,7 @@ static void pgstat_reset_after_failure(void); static bool pgstat_flush_pending_entries(bool nowait); static void pgstat_prep_snapshot(void); -static void pgstat_build_snapshot(void); +static void pgstat_build_snapshot(PgStat_Kind statKind); static void pgstat_build_snapshot_fixed(PgStat_Kind kind); static inline bool pgstat_is_kind_valid(PgStat_Kind kind); @@ -203,7 +203,7 @@ static inline bool pgstat_is_kind_valid(PgStat_Kind kind); bool pgstat_track_counts = false; int pgstat_fetch_consistency = PGSTAT_FETCH_CONSISTENCY_CACHE; - +bool pgstat_track_vacuum_statistics = true; /* ---------- * state shared with pgstat_*.c @@ -260,7 +260,6 @@ static bool pgstat_is_initialized = false; static bool pgstat_is_shutdown = false; #endif - /* * The different kinds of built-in statistics. * @@ -890,7 +889,6 @@ pgstat_reset_of_kind(PgStat_Kind kind) pgstat_reset_entries_of_kind(kind, ts); } - /* ------------------------------------------------------------ * Fetching of stats * ------------------------------------------------------------ @@ -959,7 +957,7 @@ pgstat_fetch_entry(PgStat_Kind kind, Oid dboid, uint64 objid) /* if we need to build a full snapshot, do so */ if (pgstat_fetch_consistency == PGSTAT_FETCH_CONSISTENCY_SNAPSHOT) - pgstat_build_snapshot(); + pgstat_build_snapshot(PGSTAT_KIND_INVALID); /* if caching is desired, look up in cache */ if (pgstat_fetch_consistency > PGSTAT_FETCH_CONSISTENCY_NONE) @@ -1075,7 +1073,7 @@ pgstat_snapshot_fixed(PgStat_Kind kind) pgstat_clear_snapshot(); if (pgstat_fetch_consistency == PGSTAT_FETCH_CONSISTENCY_SNAPSHOT) - pgstat_build_snapshot(); + pgstat_build_snapshot(PGSTAT_KIND_INVALID); else pgstat_build_snapshot_fixed(kind); @@ -1126,7 +1124,7 @@ pgstat_prep_snapshot(void) } static void -pgstat_build_snapshot(void) +pgstat_build_snapshot(PgStat_Kind statKind) { dshash_seq_status hstat; PgStatShared_HashEntry *p; diff --git a/src/backend/utils/activity/pgstat_relation.c b/src/backend/utils/activity/pgstat_relation.c index faba8b64d23..bbb095389be 100644 --- a/src/backend/utils/activity/pgstat_relation.c +++ b/src/backend/utils/activity/pgstat_relation.c @@ -47,6 +47,8 @@ static void add_tabstat_xact_level(PgStat_TableStatus *pgstat_info, int nest_lev static void ensure_tabstat_xact_level(PgStat_TableStatus *pgstat_info); static void save_truncdrop_counters(PgStat_TableXactStatus *trans, bool is_drop); static void restore_truncdrop_counters(PgStat_TableXactStatus *trans); +static void pgstat_accumulate_extvac_stats(ExtVacReport *dst, ExtVacReport *src, + bool accumulate_reltype_specific_info); /* @@ -208,7 +210,8 @@ pgstat_drop_relation(Relation rel) */ void pgstat_report_vacuum(Oid tableoid, bool shared, - PgStat_Counter livetuples, PgStat_Counter deadtuples) + PgStat_Counter livetuples, PgStat_Counter deadtuples, + ExtVacReport *params) { PgStat_EntryRef *entry_ref; PgStatShared_Relation *shtabentry; @@ -232,6 +235,8 @@ pgstat_report_vacuum(Oid tableoid, bool shared, tabentry->live_tuples = livetuples; tabentry->dead_tuples = deadtuples; + pgstat_accumulate_extvac_stats(&tabentry->vacuum_ext, params, true); + /* * It is quite possible that a non-aggressive VACUUM ended up skipping * various pages, however, we'll zero the insert counter here regardless. @@ -857,6 +862,9 @@ pgstat_relation_flush_cb(PgStat_EntryRef *entry_ref, bool nowait) tabentry->blocks_fetched += lstats->counts.blocks_fetched; tabentry->blocks_hit += lstats->counts.blocks_hit; + tabentry->rev_all_frozen_pages += lstats->counts.rev_all_frozen_pages; + tabentry->rev_all_visible_pages += lstats->counts.rev_all_visible_pages; + /* Clamp live_tuples in case of negative delta_live_tuples */ tabentry->live_tuples = Max(tabentry->live_tuples, 0); /* Likewise for dead_tuples */ @@ -980,3 +988,39 @@ restore_truncdrop_counters(PgStat_TableXactStatus *trans) trans->tuples_deleted = trans->deleted_pre_truncdrop; } } + +static void +pgstat_accumulate_extvac_stats(ExtVacReport *dst, ExtVacReport *src, + bool accumulate_reltype_specific_info) +{ + dst->total_blks_read += src->total_blks_read; + dst->total_blks_hit += src->total_blks_hit; + dst->total_blks_dirtied += src->total_blks_dirtied; + dst->total_blks_written += src->total_blks_written; + dst->wal_bytes += src->wal_bytes; + dst->wal_fpi += src->wal_fpi; + dst->wal_records += src->wal_records; + dst->blk_read_time += src->blk_read_time; + dst->blk_write_time += src->blk_write_time; + dst->delay_time += src->delay_time; + dst->total_time += src->total_time; + + if (!accumulate_reltype_specific_info) + return; + + dst->blks_fetched += src->blks_fetched; + dst->blks_hit += src->blks_hit; + + dst->pages_scanned += src->pages_scanned; + dst->pages_removed += src->pages_removed; + dst->vm_new_frozen_pages += src->vm_new_frozen_pages; + dst->vm_new_visible_pages += src->vm_new_visible_pages; + dst->vm_new_visible_frozen_pages += src->vm_new_visible_frozen_pages; + dst->tuples_deleted += src->tuples_deleted; + dst->tuples_frozen += src->tuples_frozen; + dst->recently_dead_tuples += src->recently_dead_tuples; + dst->index_vacuum_count += src->index_vacuum_count; + dst->missed_dead_pages += src->missed_dead_pages; + dst->missed_dead_tuples += src->missed_dead_tuples; + +} \ No newline at end of file diff --git a/src/backend/utils/adt/pgstatfuncs.c b/src/backend/utils/adt/pgstatfuncs.c index cdf37403e9d..c544f6bd73c 100644 --- a/src/backend/utils/adt/pgstatfuncs.c +++ b/src/backend/utils/adt/pgstatfuncs.c @@ -106,6 +106,12 @@ PG_STAT_GET_RELENTRY_INT64(tuples_updated) /* pg_stat_get_vacuum_count */ PG_STAT_GET_RELENTRY_INT64(vacuum_count) +/* pg_stat_get_rev_frozen_pages */ +PG_STAT_GET_RELENTRY_INT64(rev_all_frozen_pages) + +/* pg_stat_get_rev_all_visible_pages */ +PG_STAT_GET_RELENTRY_INT64(rev_all_visible_pages) + #define PG_STAT_GET_RELENTRY_TIMESTAMPTZ(stat) \ Datum \ CppConcat(pg_stat_get_,stat)(PG_FUNCTION_ARGS) \ @@ -2063,3 +2069,141 @@ pg_stat_have_stats(PG_FUNCTION_ARGS) PG_RETURN_BOOL(pgstat_have_entry(kind, dboid, objid)); } + + +/* + * Get the vacuum statistics for the heap tables. + */ +Datum +pg_stat_get_vacuum_tables(PG_FUNCTION_ARGS) +{ + #define PG_STAT_GET_VACUUM_TABLES_STATS_COLS 25 + + Oid relid = PG_GETARG_OID(0); + PgStat_StatTabEntry *tabentry; + ExtVacReport *extvacuum; + TupleDesc tupdesc; + Datum values[PG_STAT_GET_VACUUM_TABLES_STATS_COLS] = {0}; + bool nulls[PG_STAT_GET_VACUUM_TABLES_STATS_COLS] = {0}; + char buf[256]; + int i = 0; + ExtVacReport allzero; + + /* Initialise attributes information in the tuple descriptor */ + tupdesc = CreateTemplateTupleDesc(PG_STAT_GET_VACUUM_TABLES_STATS_COLS); + + TupleDescInitEntry(tupdesc, (AttrNumber) ++i, "relid", + INT4OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) ++i, "total_blks_read", + INT8OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) ++i, "total_blks_hit", + INT8OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) ++i, "total_blks_dirtied", + INT8OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) ++i, "total_blks_written", + INT8OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) ++i, "rel_blks_read", + INT8OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) ++i, "rel_blks_hit", + INT8OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) ++i, "pages_scanned", + INT8OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) ++i, "pages_removed", + INT8OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) ++i, "vm_new_frozen_pages", + INT8OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) ++i, "vm_new_visible_pages", + INT8OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) ++i, "vm_new_visible_frozen_pages", + INT8OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) ++i, "missed_dead_pages", + INT8OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) ++i, "tuples_deleted", + INT8OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) ++i, "tuples_frozen", + INT8OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) ++i, "recently_dead_tuples", + INT8OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) ++i, "missed_dead_tuples", + INT8OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) ++i, "index_vacuum_count", + INT8OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) ++i, "wal_records", + INT8OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) ++i, "wal_fpi", + INT8OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) ++i, "wal_bytes", + NUMERICOID, -1, 0); + + TupleDescInitEntry(tupdesc, (AttrNumber) ++i, "blk_read_time", + FLOAT8OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) ++i, "blk_write_time", + FLOAT8OID, -1, 0); + + TupleDescInitEntry(tupdesc, (AttrNumber) ++i, "delay_time", + FLOAT8OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) ++i, "total_time", + FLOAT8OID, -1, 0); + + Assert(i == PG_STAT_GET_VACUUM_TABLES_STATS_COLS); + + BlessTupleDesc(tupdesc); + + tabentry = pgstat_fetch_stat_tabentry(relid); + + if (tabentry == NULL) + { + /* If the subscription is not found, initialise its stats */ + memset(&allzero, 0, sizeof(ExtVacReport)); + extvacuum = &allzero; + } + else + { + extvacuum = &(tabentry->vacuum_ext); + } + + i = 0; + + values[i++] = ObjectIdGetDatum(relid); + + values[i++] = Int64GetDatum(extvacuum->total_blks_read); + values[i++] = Int64GetDatum(extvacuum->total_blks_hit); + values[i++] = Int64GetDatum(extvacuum->total_blks_dirtied); + values[i++] = Int64GetDatum(extvacuum->total_blks_written); + + values[i++] = Int64GetDatum(extvacuum->blks_fetched - + extvacuum->blks_hit); + values[i++] = Int64GetDatum(extvacuum->blks_hit); + + values[i++] = Int64GetDatum(extvacuum->pages_scanned); + values[i++] = Int64GetDatum(extvacuum->pages_removed); + values[i++] = Int64GetDatum(extvacuum->vm_new_frozen_pages); + values[i++] = Int64GetDatum(extvacuum->vm_new_visible_pages); + values[i++] = Int64GetDatum(extvacuum->vm_new_visible_frozen_pages); + values[i++] = Int64GetDatum(extvacuum->missed_dead_pages); + values[i++] = Int64GetDatum(extvacuum->tuples_deleted); + values[i++] = Int64GetDatum(extvacuum->tuples_frozen); + values[i++] = Int64GetDatum(extvacuum->recently_dead_tuples); + values[i++] = Int64GetDatum(extvacuum->missed_dead_tuples); + values[i++] = Int64GetDatum(extvacuum->index_vacuum_count); + + values[i++] = Int64GetDatum(extvacuum->wal_records); + values[i++] = Int64GetDatum(extvacuum->wal_fpi); + + /* Convert to numeric, like pg_stat_statements */ + snprintf(buf, sizeof buf, UINT64_FORMAT, extvacuum->wal_bytes); + values[i++] = DirectFunctionCall3(numeric_in, + CStringGetDatum(buf), + ObjectIdGetDatum(0), + Int32GetDatum(-1)); + + values[i++] = Float8GetDatum(extvacuum->blk_read_time); + values[i++] = Float8GetDatum(extvacuum->blk_write_time); + values[i++] = Float8GetDatum(extvacuum->delay_time); + values[i++] = Float8GetDatum(extvacuum->total_time); + + Assert(i == PG_STAT_GET_VACUUM_TABLES_STATS_COLS); + + /* Returns the record as Datum */ + PG_RETURN_DATUM(HeapTupleGetDatum(heap_form_tuple(tupdesc, values, nulls))); +} \ No newline at end of file diff --git a/src/backend/utils/error/elog.c b/src/backend/utils/error/elog.c index 289059435a9..041fd3730a2 100644 --- a/src/backend/utils/error/elog.c +++ b/src/backend/utils/error/elog.c @@ -1619,6 +1619,19 @@ getinternalerrposition(void) return edata->internalpos; } +/* + * Return elevel of errors + */ +int +geterrelevel(void) +{ + ErrorData *edata = &errordata[errordata_stack_depth]; + + /* we don't bother incrementing recursion_depth */ + CHECK_STACK_DEPTH(); + + return edata->elevel; +} /* * Functions to allow construction of error message strings separately from diff --git a/src/backend/utils/misc/guc_tables.c b/src/backend/utils/misc/guc_tables.c index 8cf1afbad20..bddfaba5b09 100644 --- a/src/backend/utils/misc/guc_tables.c +++ b/src/backend/utils/misc/guc_tables.c @@ -1478,6 +1478,15 @@ struct config_bool ConfigureNamesBool[] = false, NULL, NULL, NULL }, + { + {"track_vacuum_statistics", PGC_SUSET, STATS_CUMULATIVE, + gettext_noop("Collects vacuum statistics for table relations."), + NULL + }, + &pgstat_track_vacuum_statistics, + true, + NULL, NULL, NULL + }, { {"track_wal_io_timing", PGC_SUSET, STATS_CUMULATIVE, gettext_noop("Collects timing statistics for WAL I/O activity."), diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index 0f22c217235..efa60fa3bc7 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -12416,4 +12416,22 @@ proargtypes => 'int2', prosrc => 'gist_stratnum_identity' }, +{ oid => '8001', + descr => 'pg_stat_get_vacuum_tables return stats values', + proname => 'pg_stat_get_vacuum_tables', prorows => 1000, provolatile => 's', prorettype => 'record',proisstrict => 'f', + proretset => 't', + proargtypes => 'oid', + proallargtypes => '{oid,oid,int8,int8,int8,int8,int8,int8,int8,int8,int8,int8,int8,int8,int8,int8,int8,int8,int8,int8,int8,numeric,float8,float8,float8,float8}', + proargmodes => '{i,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o}', + proargnames => '{reloid,relid,total_blks_read,total_blks_hit,total_blks_dirtied,total_blks_written,rel_blks_read,rel_blks_hit,pages_scanned,pages_removed,vm_new_frozen_pages,vm_new_visible_pages,vm_new_visible_frozen_pages,missed_dead_pages,tuples_deleted,tuples_frozen,recently_dead_tuples,missed_dead_tuples,index_vacuum_count,wal_records,wal_fpi,wal_bytes,blk_read_time,blk_write_time,delay_time,total_time}', + prosrc => 'pg_stat_get_vacuum_tables' }, + + { oid => '8002', descr => 'statistics: number of times the all-visible pages in the visibility map was removed for pages of table', + proname => 'pg_stat_get_rev_all_visible_pages', provolatile => 's', + proparallel => 'r', prorettype => 'int8', proargtypes => 'oid', + prosrc => 'pg_stat_get_rev_all_visible_pages' }, + { oid => '8003', descr => 'statistics: number of times the all-frozen pages in the visibility map was removed for pages of table', + proname => 'pg_stat_get_rev_all_frozen_pages', provolatile => 's', + proparallel => 'r', prorettype => 'int8', proargtypes => 'oid', + prosrc => 'pg_stat_get_rev_all_frozen_pages' }, ] diff --git a/src/include/commands/vacuum.h b/src/include/commands/vacuum.h index 759f9a87d38..07b28b15d9f 100644 --- a/src/include/commands/vacuum.h +++ b/src/include/commands/vacuum.h @@ -308,6 +308,7 @@ extern PGDLLIMPORT int vacuum_multixact_failsafe_age; extern PGDLLIMPORT pg_atomic_uint32 *VacuumSharedCostBalance; extern PGDLLIMPORT pg_atomic_uint32 *VacuumActiveNWorkers; extern PGDLLIMPORT int VacuumCostBalanceLocal; +extern PGDLLIMPORT double VacuumDelayTime; extern PGDLLIMPORT bool VacuumFailsafeActive; extern PGDLLIMPORT double vacuum_cost_delay; diff --git a/src/include/pgstat.h b/src/include/pgstat.h index ebfeef2f460..a0c3253916a 100644 --- a/src/include/pgstat.h +++ b/src/include/pgstat.h @@ -166,6 +166,52 @@ typedef struct PgStat_BackendSubEntry PgStat_Counter conflict_count[CONFLICT_NUM_TYPES]; } PgStat_BackendSubEntry; +/* ---------- + * + * ExtVacReport + * + * Additional statistics of vacuum processing over a heap relation. + * pages_removed is the amount by which the physically shrank, + * if any (ie the change in its total size on disk) + * pages_deleted refer to free space within the index file + * ---------- + */ +typedef struct ExtVacReport +{ + /* number of blocks missed, hit, dirtied and written during a vacuum of specific relation */ + int64 total_blks_read; + int64 total_blks_hit; + int64 total_blks_dirtied; + int64 total_blks_written; + + /* blocks missed and hit for just the heap during a vacuum of specific relation */ + int64 blks_fetched; + int64 blks_hit; + + /* Vacuum WAL usage stats */ + int64 wal_records; /* wal usage: number of WAL records */ + int64 wal_fpi; /* wal usage: number of WAL full page images produced */ + uint64 wal_bytes; /* wal usage: size of WAL records produced */ + + /* Time stats. */ + double blk_read_time; /* time spent reading pages, in msec */ + double blk_write_time; /* time spent writing pages, in msec */ + double delay_time; /* how long vacuum slept in vacuum delay point, in msec */ + double total_time; /* total time of a vacuum operation, in msec */ + + int64 pages_scanned; /* heap pages examined (not skipped by VM) */ + int64 pages_removed; /* heap pages removed by vacuum "truncation" */ + int64 vm_new_frozen_pages; /* pages marked in VM as frozen */ + int64 vm_new_visible_pages; /* pages marked in VM as all-visible */ + int64 vm_new_visible_frozen_pages; /* pages marked in VM as all-visible and frozen */ + int64 missed_dead_tuples; /* tuples not pruned by vacuum due to failure to get a cleanup lock */ + int64 missed_dead_pages; /* pages with missed dead tuples */ + int64 tuples_deleted; /* tuples deleted by vacuum */ + int64 tuples_frozen; /* tuples frozen up by vacuum */ + int64 recently_dead_tuples; /* deleted tuples that are still visible to some transaction */ + int64 index_vacuum_count; /* number of index vacuumings */ +} ExtVacReport; + /* ---------- * PgStat_TableCounts The actual per-table counts kept by a backend * @@ -208,6 +254,16 @@ typedef struct PgStat_TableCounts PgStat_Counter blocks_fetched; PgStat_Counter blocks_hit; + + PgStat_Counter rev_all_visible_pages; + PgStat_Counter rev_all_frozen_pages; + + /* + * Additional cumulative stat on vacuum operations. + * Use an expensive structure as an abstraction for different types of + * relations. + */ + ExtVacReport vacuum_ext; } PgStat_TableCounts; /* ---------- @@ -266,7 +322,7 @@ typedef struct PgStat_TableXactStatus * ------------------------------------------------------------ */ -#define PGSTAT_FILE_FORMAT_ID 0x01A5BCAF +#define PGSTAT_FILE_FORMAT_ID 0x01A5BCB0 typedef struct PgStat_ArchiverStats { @@ -405,6 +461,8 @@ typedef struct PgStat_StatDBEntry PgStat_Counter parallel_workers_launched; TimestampTz stat_reset_timestamp; + + ExtVacReport vacuum_ext; /* extended vacuum statistics */ } PgStat_StatDBEntry; typedef struct PgStat_StatFuncEntry @@ -478,6 +536,11 @@ typedef struct PgStat_StatTabEntry PgStat_Counter analyze_count; TimestampTz last_autoanalyze_time; /* autovacuum initiated */ PgStat_Counter autoanalyze_count; + + PgStat_Counter rev_all_visible_pages; + PgStat_Counter rev_all_frozen_pages; + + ExtVacReport vacuum_ext; } PgStat_StatTabEntry; typedef struct PgStat_WalStats @@ -645,7 +708,8 @@ extern void pgstat_assoc_relation(Relation rel); extern void pgstat_unlink_relation(Relation rel); extern void pgstat_report_vacuum(Oid tableoid, bool shared, - PgStat_Counter livetuples, PgStat_Counter deadtuples); + PgStat_Counter livetuples, PgStat_Counter deadtuples, + ExtVacReport *params); extern void pgstat_report_analyze(Relation rel, PgStat_Counter livetuples, PgStat_Counter deadtuples, bool resetcounter); @@ -696,6 +760,17 @@ extern void pgstat_report_analyze(Relation rel, if (pgstat_should_count_relation(rel)) \ (rel)->pgstat_info->counts.blocks_hit++; \ } while (0) +/* accumulate unfrozen all-visible and all-frozen pages */ +#define pgstat_count_vm_rev_all_visible(rel) \ + do { \ + if (pgstat_should_count_relation(rel)) \ + (rel)->pgstat_info->counts.rev_all_visible_pages++; \ + } while (0) +#define pgstat_count_vm_rev_all_frozen(rel) \ + do { \ + if (pgstat_should_count_relation(rel)) \ + (rel)->pgstat_info->counts.rev_all_frozen_pages++; \ + } while (0) extern void pgstat_count_heap_insert(Relation rel, PgStat_Counter n); extern void pgstat_count_heap_update(Relation rel, bool hot, bool newpage); @@ -713,7 +788,6 @@ extern PgStat_StatTabEntry *pgstat_fetch_stat_tabentry_ext(bool shared, Oid reloid); extern PgStat_TableStatus *find_tabstat_entry(Oid rel_id); - /* * Functions in pgstat_replslot.c */ @@ -784,6 +858,7 @@ extern PgStat_WalStats *pgstat_fetch_stat_wal(void); extern PGDLLIMPORT bool pgstat_track_counts; extern PGDLLIMPORT int pgstat_track_functions; extern PGDLLIMPORT int pgstat_fetch_consistency; +extern PGDLLIMPORT bool pgstat_track_vacuum_statistics; /* diff --git a/src/include/utils/elog.h b/src/include/utils/elog.h index e54eca5b489..e752c0ce015 100644 --- a/src/include/utils/elog.h +++ b/src/include/utils/elog.h @@ -230,6 +230,7 @@ extern int geterrlevel(void); extern int geterrposition(void); extern int getinternalerrposition(void); +extern int geterrelevel(void); /*---------- * Old-style error reporting API: to be used in this way: diff --git a/src/include/utils/pgstat_internal.h b/src/include/utils/pgstat_internal.h index 7338bc1e28e..d9328be48a7 100644 --- a/src/include/utils/pgstat_internal.h +++ b/src/include/utils/pgstat_internal.h @@ -595,7 +595,7 @@ extern PgStat_EntryRef *pgstat_fetch_pending_entry(PgStat_Kind kind, extern void *pgstat_fetch_entry(PgStat_Kind kind, Oid dboid, uint64 objid); extern void pgstat_snapshot_fixed(PgStat_Kind kind); - +extern void pgstat_update_snapshot(PgStat_Kind kind); /* * Functions in pgstat_archiver.c */ diff --git a/src/test/isolation/expected/vacuum-extending-in-repetable-read.out b/src/test/isolation/expected/vacuum-extending-in-repetable-read.out new file mode 100644 index 00000000000..87f7e40b4a6 --- /dev/null +++ b/src/test/isolation/expected/vacuum-extending-in-repetable-read.out @@ -0,0 +1,53 @@ +unused step name: s2_delete +Parsed test spec with 2 sessions + +starting permutation: s2_insert s2_print_vacuum_stats_table s1_begin_repeatable_read s2_update s2_insert_interrupt s2_vacuum s2_print_vacuum_stats_table s1_commit s2_checkpoint s2_vacuum s2_print_vacuum_stats_table +step s2_insert: INSERT INTO test_vacuum_stat_isolation(id, ival) SELECT ival, ival%10 FROM generate_series(1,1000) As ival; +step s2_print_vacuum_stats_table: + SELECT + vt.relname, vt.tuples_deleted, vt.recently_dead_tuples, vt.missed_dead_tuples, vt.missed_dead_pages, vt.tuples_frozen + FROM pg_stat_vacuum_tables vt, pg_class c + WHERE vt.relname = 'test_vacuum_stat_isolation' AND vt.relid = c.oid; + +relname |tuples_deleted|recently_dead_tuples|missed_dead_tuples|missed_dead_pages|tuples_frozen +--------------------------+--------------+--------------------+------------------+-----------------+------------- +test_vacuum_stat_isolation| 0| 0| 0| 0| 0 +(1 row) + +step s1_begin_repeatable_read: + BEGIN transaction ISOLATION LEVEL REPEATABLE READ; + select count(ival) from test_vacuum_stat_isolation where id>900; + +count +----- + 100 +(1 row) + +step s2_update: UPDATE test_vacuum_stat_isolation SET ival = ival + 2 where id > 900; +step s2_insert_interrupt: INSERT INTO test_vacuum_stat_isolation values (1,1); +step s2_vacuum: VACUUM test_vacuum_stat_isolation; +step s2_print_vacuum_stats_table: + SELECT + vt.relname, vt.tuples_deleted, vt.recently_dead_tuples, vt.missed_dead_tuples, vt.missed_dead_pages, vt.tuples_frozen + FROM pg_stat_vacuum_tables vt, pg_class c + WHERE vt.relname = 'test_vacuum_stat_isolation' AND vt.relid = c.oid; + +relname |tuples_deleted|recently_dead_tuples|missed_dead_tuples|missed_dead_pages|tuples_frozen +--------------------------+--------------+--------------------+------------------+-----------------+------------- +test_vacuum_stat_isolation| 0| 100| 0| 0| 0 +(1 row) + +step s1_commit: COMMIT; +step s2_checkpoint: CHECKPOINT; +step s2_vacuum: VACUUM test_vacuum_stat_isolation; +step s2_print_vacuum_stats_table: + SELECT + vt.relname, vt.tuples_deleted, vt.recently_dead_tuples, vt.missed_dead_tuples, vt.missed_dead_pages, vt.tuples_frozen + FROM pg_stat_vacuum_tables vt, pg_class c + WHERE vt.relname = 'test_vacuum_stat_isolation' AND vt.relid = c.oid; + +relname |tuples_deleted|recently_dead_tuples|missed_dead_tuples|missed_dead_pages|tuples_frozen +--------------------------+--------------+--------------------+------------------+-----------------+------------- +test_vacuum_stat_isolation| 100| 100| 0| 0| 101 +(1 row) + diff --git a/src/test/isolation/isolation_schedule b/src/test/isolation/isolation_schedule index 143109aa4da..e93dd4f626c 100644 --- a/src/test/isolation/isolation_schedule +++ b/src/test/isolation/isolation_schedule @@ -95,6 +95,7 @@ test: timeouts test: vacuum-concurrent-drop test: vacuum-conflict test: vacuum-skip-locked +test: vacuum-extending-in-repetable-read test: stats test: horizons test: predicate-hash diff --git a/src/test/isolation/specs/vacuum-extending-in-repetable-read.spec b/src/test/isolation/specs/vacuum-extending-in-repetable-read.spec new file mode 100644 index 00000000000..5893d89573d --- /dev/null +++ b/src/test/isolation/specs/vacuum-extending-in-repetable-read.spec @@ -0,0 +1,53 @@ +# Test for checking recently_dead_tuples, tuples_deleted and frozen tuples in pg_stat_vacuum_tables. +# recently_dead_tuples values are counted when vacuum hasn't cleared tuples because they were deleted recently. +# recently_dead_tuples aren't increased after releasing lock compared with tuples_deleted, which increased +# by the value of the cleared tuples that the vacuum managed to clear. + +setup +{ + CREATE TABLE test_vacuum_stat_isolation(id int, ival int) WITH (autovacuum_enabled = off); + SET track_io_timing = on; + SET track_vacuum_statistics TO 'on'; +} + +teardown +{ + DROP TABLE test_vacuum_stat_isolation CASCADE; + RESET track_io_timing; + RESET track_vacuum_statistics; +} + +session s1 +step s1_begin_repeatable_read { + BEGIN transaction ISOLATION LEVEL REPEATABLE READ; + select count(ival) from test_vacuum_stat_isolation where id>900; + } +step s1_commit { COMMIT; } + +session s2 +step s2_insert { INSERT INTO test_vacuum_stat_isolation(id, ival) SELECT ival, ival%10 FROM generate_series(1,1000) As ival; } +step s2_update { UPDATE test_vacuum_stat_isolation SET ival = ival + 2 where id > 900; } +step s2_delete { DELETE FROM test_vacuum_stat_isolation where id > 900; } +step s2_insert_interrupt { INSERT INTO test_vacuum_stat_isolation values (1,1); } +step s2_vacuum { VACUUM test_vacuum_stat_isolation; } +step s2_checkpoint { CHECKPOINT; } +step s2_print_vacuum_stats_table +{ + SELECT + vt.relname, vt.tuples_deleted, vt.recently_dead_tuples, vt.missed_dead_tuples, vt.missed_dead_pages, vt.tuples_frozen + FROM pg_stat_vacuum_tables vt, pg_class c + WHERE vt.relname = 'test_vacuum_stat_isolation' AND vt.relid = c.oid; +} + +permutation + s2_insert + s2_print_vacuum_stats_table + s1_begin_repeatable_read + s2_update + s2_insert_interrupt + s2_vacuum + s2_print_vacuum_stats_table + s1_commit + s2_checkpoint + s2_vacuum + s2_print_vacuum_stats_table \ No newline at end of file diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out index 3014d047fef..dd795d58dfc 100644 --- a/src/test/regress/expected/rules.out +++ b/src/test/regress/expected/rules.out @@ -1804,7 +1804,9 @@ pg_stat_all_tables| SELECT c.oid AS relid, pg_stat_get_vacuum_count(c.oid) AS vacuum_count, pg_stat_get_autovacuum_count(c.oid) AS autovacuum_count, pg_stat_get_analyze_count(c.oid) AS analyze_count, - pg_stat_get_autoanalyze_count(c.oid) AS autoanalyze_count + pg_stat_get_autoanalyze_count(c.oid) AS autoanalyze_count, + pg_stat_get_rev_all_frozen_pages(c.oid) AS rev_all_frozen_pages, + pg_stat_get_rev_all_visible_pages(c.oid) AS rev_all_visible_pages FROM ((pg_class c LEFT JOIN pg_index i ON ((c.oid = i.indrelid))) LEFT JOIN pg_namespace n ON ((n.oid = c.relnamespace))) @@ -2188,7 +2190,9 @@ pg_stat_sys_tables| SELECT relid, vacuum_count, autovacuum_count, analyze_count, - autoanalyze_count + autoanalyze_count, + rev_all_frozen_pages, + rev_all_visible_pages FROM pg_stat_all_tables WHERE ((schemaname = ANY (ARRAY['pg_catalog'::name, 'information_schema'::name])) OR (schemaname ~ '^pg_toast'::text)); pg_stat_user_functions| SELECT p.oid AS funcid, @@ -2236,9 +2240,42 @@ pg_stat_user_tables| SELECT relid, vacuum_count, autovacuum_count, analyze_count, - autoanalyze_count + autoanalyze_count, + rev_all_frozen_pages, + rev_all_visible_pages FROM pg_stat_all_tables WHERE ((schemaname <> ALL (ARRAY['pg_catalog'::name, 'information_schema'::name])) AND (schemaname !~ '^pg_toast'::text)); +pg_stat_vacuum_tables| SELECT ns.nspname AS schema, + rel.relname, + stats.relid, + stats.total_blks_read, + stats.total_blks_hit, + stats.total_blks_dirtied, + stats.total_blks_written, + stats.rel_blks_read, + stats.rel_blks_hit, + stats.pages_scanned, + stats.pages_removed, + stats.vm_new_frozen_pages, + stats.vm_new_visible_pages, + stats.vm_new_visible_frozen_pages, + stats.missed_dead_pages, + stats.tuples_deleted, + stats.tuples_frozen, + stats.recently_dead_tuples, + stats.missed_dead_tuples, + stats.index_vacuum_count, + stats.wal_records, + stats.wal_fpi, + stats.wal_bytes, + stats.blk_read_time, + stats.blk_write_time, + stats.delay_time, + stats.total_time + FROM (pg_class rel + JOIN pg_namespace ns ON ((ns.oid = rel.relnamespace))), + LATERAL pg_stat_get_vacuum_tables(rel.oid) stats(relid, total_blks_read, total_blks_hit, total_blks_dirtied, total_blks_written, rel_blks_read, rel_blks_hit, pages_scanned, pages_removed, vm_new_frozen_pages, vm_new_visible_pages, vm_new_visible_frozen_pages, missed_dead_pages, tuples_deleted, tuples_frozen, recently_dead_tuples, missed_dead_tuples, index_vacuum_count, wal_records, wal_fpi, wal_bytes, blk_read_time, blk_write_time, delay_time, total_time) + WHERE (rel.relkind = 'r'::"char"); pg_stat_wal| SELECT wal_records, wal_fpi, wal_bytes, diff --git a/src/test/regress/expected/vacuum_tables_statistics.out b/src/test/regress/expected/vacuum_tables_statistics.out new file mode 100644 index 00000000000..0c05a812dd1 --- /dev/null +++ b/src/test/regress/expected/vacuum_tables_statistics.out @@ -0,0 +1,225 @@ +-- +-- Test cumulative vacuum stats system +-- +-- Check the wall statistics collected during vacuum operation: +-- number of frozen and visible pages set by vacuum; +-- number of frozen and visible pages removed by backend. +-- Statistic wal_fpi is not displayed in this test because its behavior is unstable. +-- +-- conditio sine qua non +SHOW track_counts; -- must be on + track_counts +-------------- + on +(1 row) + +\set sample_size 10000 +-- not enabled by default, but we want to test it... +SET track_functions TO 'all'; +-- Test that vacuum statistics will be empty when parameter is off. +SET track_vacuum_statistics TO 'off'; +CREATE TABLE vestat (x int) WITH (autovacuum_enabled = off, fillfactor = 10); +INSERT INTO vestat SELECT x FROM generate_series(1,:sample_size) as x; +ANALYZE vestat; +DELETE FROM vestat WHERE x % 2 = 0; +VACUUM (PARALLEL 0, BUFFER_USAGE_LIMIT 128) vestat; +-- Must be empty. +SELECT relname,total_blks_read, total_blks_hit, total_blks_dirtied, total_blks_written,rel_blks_read, rel_blks_hit, +pages_scanned, pages_removed, vm_new_frozen_pages, vm_new_visible_pages, vm_new_visible_frozen_pages, missed_dead_pages, +tuples_deleted, tuples_frozen, recently_dead_tuples, missed_dead_tuples, index_vacuum_count, +wal_records, wal_fpi, wal_bytes, blk_read_time, blk_write_time,delay_time, total_time +FROM pg_stat_vacuum_tables vt +WHERE vt.relname = 'vestat'; + relname | total_blks_read | total_blks_hit | total_blks_dirtied | total_blks_written | rel_blks_read | rel_blks_hit | pages_scanned | pages_removed | vm_new_frozen_pages | vm_new_visible_pages | vm_new_visible_frozen_pages | missed_dead_pages | tuples_deleted | tuples_frozen | recently_dead_tuples | missed_dead_tuples | index_vacuum_count | wal_records | wal_fpi | wal_bytes | blk_read_time | blk_write_time | delay_time | total_time +---------+-----------------+----------------+--------------------+--------------------+---------------+--------------+---------------+---------------+---------------------+----------------------+-----------------------------+-------------------+----------------+---------------+----------------------+--------------------+--------------------+-------------+---------+-----------+---------------+----------------+------------+------------ + vestat | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 +(1 row) + +RESET track_vacuum_statistics; +DROP TABLE vestat CASCADE; +SHOW track_vacuum_statistics; -- must be on + track_vacuum_statistics +------------------------- + on +(1 row) + +-- ensure pending stats are flushed +SELECT pg_stat_force_next_flush(); + pg_stat_force_next_flush +-------------------------- + +(1 row) + +--SET stats_fetch_consistency = snapshot; +CREATE TABLE vestat (x int) WITH (autovacuum_enabled = off, fillfactor = 10); +INSERT INTO vestat SELECT x FROM generate_series(1,:sample_size) as x; +ANALYZE vestat; +SELECT oid AS roid from pg_class where relname = 'vestat' \gset +DELETE FROM vestat WHERE x % 2 = 0; +-- Before the first vacuum execution extended stats view is empty. +SELECT vt.relname,vm_new_frozen_pages,tuples_deleted,relpages,pages_scanned,pages_removed +FROM pg_stat_vacuum_tables vt, pg_class c +WHERE vt.relname = 'vestat' AND vt.relid = c.oid; + relname | vm_new_frozen_pages | tuples_deleted | relpages | pages_scanned | pages_removed +---------+---------------------+----------------+----------+---------------+--------------- + vestat | 0 | 0 | 455 | 0 | 0 +(1 row) + +SELECT relpages AS rp +FROM pg_class c +WHERE relname = 'vestat' \gset +VACUUM (PARALLEL 0, BUFFER_USAGE_LIMIT 128, INDEX_CLEANUP OFF) vestat; +-- it is necessary to check the wal statistics +CHECKPOINT; +-- The table and index extended vacuum statistics should show us that +-- vacuum frozed pages and clean up pages, but pages_removed stayed the same +-- because of not full table have cleaned up +SELECT vt.relname,vm_new_frozen_pages > 0 AS vm_new_frozen_pages,tuples_deleted > 0 AS tuples_deleted,relpages-:rp = 0 AS relpages,pages_scanned > 0 AS pages_scanned,pages_removed = 0 AS pages_removed +FROM pg_stat_vacuum_tables vt, pg_class c +WHERE vt.relname = 'vestat' AND vt.relid = c.oid; + relname | vm_new_frozen_pages | tuples_deleted | relpages | pages_scanned | pages_removed +---------+---------------------+----------------+----------+---------------+--------------- + vestat | f | t | t | t | t +(1 row) + +SELECT vm_new_frozen_pages AS fp,tuples_deleted AS td,relpages AS rp, pages_scanned AS ps, pages_removed AS pr +FROM pg_stat_vacuum_tables vt, pg_class c +WHERE vt.relname = 'vestat' AND vt.relid = c.oid \gset +-- Store WAL advances into variables +SELECT wal_records AS hwr,wal_bytes AS hwb,wal_fpi AS hfpi FROM pg_stat_vacuum_tables WHERE relname = 'vestat' \gset +-- Look into WAL records deltas. +SELECT wal_records > 0 AS dWR, wal_bytes > 0 AS dWB +FROM pg_stat_vacuum_tables WHERE relname = 'vestat'; + dwr | dwb +-----+----- + t | t +(1 row) + +DELETE FROM vestat;; +VACUUM (PARALLEL 0, BUFFER_USAGE_LIMIT 128, INDEX_CLEANUP OFF) vestat; +-- it is necessary to check the wal statistics +CHECKPOINT; +-- pages_removed must be increased +SELECT vt.relname,vm_new_frozen_pages-:fp > 0 AS vm_new_frozen_pages,tuples_deleted-:td > 0 AS tuples_deleted,relpages -:rp = 0 AS relpages,pages_scanned-:ps > 0 AS pages_scanned,pages_removed-:pr > 0 AS pages_removed +FROM pg_stat_vacuum_tables vt, pg_class c +WHERE vt.relname = 'vestat' AND vt.relid = c.oid; + relname | vm_new_frozen_pages | tuples_deleted | relpages | pages_scanned | pages_removed +---------+---------------------+----------------+----------+---------------+--------------- + vestat | f | t | f | t | t +(1 row) + +SELECT vm_new_frozen_pages AS fp,tuples_deleted AS td,relpages AS rp, pages_scanned AS ps, pages_removed AS pr +FROM pg_stat_vacuum_tables vt, pg_class c +WHERE vt.relname = 'vestat' AND vt.relid = c.oid \gset +-- Store WAL advances into variables +SELECT wal_records-:hwr AS dwr, wal_bytes-:hwb AS dwb, wal_fpi-:hfpi AS dfpi +FROM pg_stat_vacuum_tables WHERE relname = 'vestat' \gset +-- WAL advance should be detected. +SELECT :dwr > 0 AS dWR, :dwb > 0 AS dWB; + dwr | dwb +-----+----- + t | t +(1 row) + +-- Store WAL advances into variables +SELECT wal_records AS hwr,wal_bytes AS hwb,wal_fpi AS hfpi FROM pg_stat_vacuum_tables WHERE relname = 'vestat' \gset +INSERT INTO vestat SELECT x FROM generate_series(1,:sample_size) as x; +DELETE FROM vestat WHERE x % 2 = 0; +-- VACUUM FULL doesn't report to stat collector. So, no any advancements of statistics +-- are detected here. +VACUUM FULL vestat; +-- It is necessary to check the wal statistics +CHECKPOINT; +-- Store WAL advances into variables +SELECT wal_records-:hwr AS dwr2, wal_bytes-:hwb AS dwb2, wal_fpi-:hfpi AS dfpi2 +FROM pg_stat_vacuum_tables WHERE relname = 'vestat' \gset +-- WAL and other statistics advance should not be detected. +SELECT :dwr2=0 AS dWR, :dfpi2=0 AS dFPI, :dwb2=0 AS dWB; + dwr | dfpi | dwb +-----+------+----- + t | t | t +(1 row) + +SELECT vt.relname,vm_new_frozen_pages-:fp = 0 AS vm_new_frozen_pages,tuples_deleted-:td = 0 AS tuples_deleted,relpages -:rp < 0 AS relpages,pages_scanned-:ps = 0 AS pages_scanned,pages_removed-:pr = 0 AS pages_removed +FROM pg_stat_vacuum_tables vt, pg_class c +WHERE vt.relname = 'vestat' AND vt.relid = c.oid; + relname | vm_new_frozen_pages | tuples_deleted | relpages | pages_scanned | pages_removed +---------+---------------------+----------------+----------+---------------+--------------- + vestat | t | t | f | t | t +(1 row) + +SELECT vm_new_frozen_pages AS fp,tuples_deleted AS td,relpages AS rp, pages_scanned AS ps,pages_removed AS pr +FROM pg_stat_vacuum_tables vt, pg_class c +WHERE vt.relname = 'vestat' AND vt.relid = c.oid \gset +-- Store WAL advances into variables +SELECT wal_records AS hwr,wal_bytes AS hwb,wal_fpi AS hfpi FROM pg_stat_vacuum_tables WHERE relname = 'vestat' \gset +DELETE FROM vestat; +TRUNCATE vestat; +VACUUM (PARALLEL 0, BUFFER_USAGE_LIMIT 128, INDEX_CLEANUP OFF) vestat; +-- it is necessary to check the wal statistics +CHECKPOINT; +-- Store WAL advances into variables after removing all tuples from the table +SELECT wal_records-:hwr AS dwr3, wal_bytes-:hwb AS dwb3, wal_fpi-:hfpi AS dfpi3 +FROM pg_stat_vacuum_tables WHERE relname = 'vestat' \gset +--There are nothing changed +SELECT :dwr3>0 AS dWR, :dfpi3=0 AS dFPI, :dwb3>0 AS dWB; + dwr | dfpi | dwb +-----+------+----- + t | t | t +(1 row) + +-- +-- Now, the table and index is compressed into zero number of pages. Check it +-- in vacuum extended statistics. +-- The vm_new_frozen_pages, pages_scanned values shouldn't be changed +-- +SELECT vt.relname,vm_new_frozen_pages-:fp = 0 AS vm_new_frozen_pages,tuples_deleted-:td = 0 AS tuples_deleted,relpages -:rp = 0 AS relpages,pages_scanned-:ps = 0 AS pages_scanned,pages_removed-:pr = 0 AS pages_removed +FROM pg_stat_vacuum_tables vt, pg_class c +WHERE vt.relname = 'vestat' AND vt.relid = c.oid; + relname | vm_new_frozen_pages | tuples_deleted | relpages | pages_scanned | pages_removed +---------+---------------------+----------------+----------+---------------+--------------- + vestat | t | t | f | t | t +(1 row) + +INSERT INTO vestat SELECT x FROM generate_series(1,:sample_size) as x; +ANALYZE vestat; +-- must be empty +SELECT vm_new_frozen_pages, vm_new_visible_pages, rev_all_frozen_pages,rev_all_visible_pages,vm_new_visible_frozen_pages +FROM pg_stat_vacuum_tables, pg_stat_all_tables WHERE pg_stat_vacuum_tables.relname = 'vestat' and pg_stat_vacuum_tables.relid = pg_stat_all_tables.relid; + vm_new_frozen_pages | vm_new_visible_pages | rev_all_frozen_pages | rev_all_visible_pages | vm_new_visible_frozen_pages +---------------------+----------------------+----------------------+-----------------------+----------------------------- + 0 | 910 | 0 | 0 | 455 +(1 row) + +VACUUM (PARALLEL 0, BUFFER_USAGE_LIMIT 128) vestat; +-- backend defreezed pages +SELECT vm_new_frozen_pages > 0 AS vm_new_frozen_pages,vm_new_visible_pages > 0 AS vm_new_visible_pages,vm_new_visible_frozen_pages > 0 AS vm_new_visible_frozen_pages,rev_all_frozen_pages = 0 AS rev_all_frozen_pages,rev_all_visible_pages = 0 AS rev_all_visible_pages +FROM pg_stat_vacuum_tables, pg_stat_all_tables WHERE pg_stat_vacuum_tables.relname = 'vestat' and pg_stat_vacuum_tables.relid = pg_stat_all_tables.relid; + vm_new_frozen_pages | vm_new_visible_pages | vm_new_visible_frozen_pages | rev_all_frozen_pages | rev_all_visible_pages +---------------------+----------------------+-----------------------------+----------------------+----------------------- + f | t | t | t | t +(1 row) + +SELECT vm_new_frozen_pages AS pf, vm_new_visible_pages AS pv,vm_new_visible_frozen_pages AS pvf, rev_all_frozen_pages AS hafp,rev_all_visible_pages AS havp +FROM pg_stat_vacuum_tables, pg_stat_all_tables WHERE pg_stat_vacuum_tables.relname = 'vestat' and pg_stat_vacuum_tables.relid = pg_stat_all_tables.relid \gset +UPDATE vestat SET x = x + 1001; +VACUUM (PARALLEL 0, BUFFER_USAGE_LIMIT 128) vestat; +SELECT vm_new_frozen_pages > :pf AS vm_new_frozen_pages,vm_new_visible_pages > :pv AS vm_new_visible_pages,vm_new_visible_frozen_pages > :pvf AS vm_new_visible_frozen_pages,rev_all_frozen_pages > :hafp AS rev_all_frozen_pages,rev_all_visible_pages > :havp AS rev_all_visible_pages +FROM pg_stat_vacuum_tables, pg_stat_all_tables WHERE pg_stat_vacuum_tables.relname = 'vestat' and pg_stat_vacuum_tables.relid = pg_stat_all_tables.relid; + vm_new_frozen_pages | vm_new_visible_pages | vm_new_visible_frozen_pages | rev_all_frozen_pages | rev_all_visible_pages +---------------------+----------------------+-----------------------------+----------------------+----------------------- + f | t | f | f | f +(1 row) + +SELECT vm_new_frozen_pages AS pf, vm_new_visible_pages AS pv, vm_new_visible_frozen_pages AS pvf, rev_all_frozen_pages AS hafp,rev_all_visible_pages AS havp +FROM pg_stat_vacuum_tables, pg_stat_all_tables WHERE pg_stat_vacuum_tables.relname = 'vestat' and pg_stat_vacuum_tables.relid = pg_stat_all_tables.relid \gset +VACUUM (PARALLEL 0, BUFFER_USAGE_LIMIT 128) vestat; +-- vacuum freezed pages +SELECT vm_new_frozen_pages = :pf AS vm_new_frozen_pages,vm_new_visible_pages = :pv AS vm_new_visible_pages,vm_new_visible_frozen_pages = :pvf AS vm_new_visible_frozen_pages, rev_all_frozen_pages = :hafp AS rev_all_frozen_pages,rev_all_visible_pages = :havp AS rev_all_visible_pages +FROM pg_stat_vacuum_tables, pg_stat_all_tables WHERE pg_stat_vacuum_tables.relname = 'vestat' and pg_stat_vacuum_tables.relid = pg_stat_all_tables.relid; + vm_new_frozen_pages | vm_new_visible_pages | vm_new_visible_frozen_pages | rev_all_frozen_pages | rev_all_visible_pages +---------------------+----------------------+-----------------------------+----------------------+----------------------- + t | t | t | t | t +(1 row) + +DROP TABLE vestat CASCADE; diff --git a/src/test/regress/parallel_schedule b/src/test/regress/parallel_schedule index 1edd9e45ebb..d11f6b7ef4b 100644 --- a/src/test/regress/parallel_schedule +++ b/src/test/regress/parallel_schedule @@ -136,3 +136,8 @@ test: fast_default # run tablespace test at the end because it drops the tablespace created during # setup that other tests may use. test: tablespace + +# ---------- +# Check vacuum statistics +# ---------- +test: vacuum_tables_statistics \ No newline at end of file diff --git a/src/test/regress/sql/vacuum_tables_statistics.sql b/src/test/regress/sql/vacuum_tables_statistics.sql new file mode 100644 index 00000000000..8ad69108ca1 --- /dev/null +++ b/src/test/regress/sql/vacuum_tables_statistics.sql @@ -0,0 +1,180 @@ +-- +-- Test cumulative vacuum stats system +-- +-- Check the wall statistics collected during vacuum operation: +-- number of frozen and visible pages set by vacuum; +-- number of frozen and visible pages removed by backend. +-- Statistic wal_fpi is not displayed in this test because its behavior is unstable. +-- + +-- conditio sine qua non +SHOW track_counts; -- must be on +\set sample_size 10000 + +-- not enabled by default, but we want to test it... +SET track_functions TO 'all'; + +-- Test that vacuum statistics will be empty when parameter is off. +SET track_vacuum_statistics TO 'off'; + +CREATE TABLE vestat (x int) WITH (autovacuum_enabled = off, fillfactor = 10); +INSERT INTO vestat SELECT x FROM generate_series(1,:sample_size) as x; +ANALYZE vestat; + +DELETE FROM vestat WHERE x % 2 = 0; + +VACUUM (PARALLEL 0, BUFFER_USAGE_LIMIT 128) vestat; + +-- Must be empty. +SELECT relname,total_blks_read, total_blks_hit, total_blks_dirtied, total_blks_written,rel_blks_read, rel_blks_hit, +pages_scanned, pages_removed, vm_new_frozen_pages, vm_new_visible_pages, vm_new_visible_frozen_pages, missed_dead_pages, +tuples_deleted, tuples_frozen, recently_dead_tuples, missed_dead_tuples, index_vacuum_count, +wal_records, wal_fpi, wal_bytes, blk_read_time, blk_write_time,delay_time, total_time +FROM pg_stat_vacuum_tables vt +WHERE vt.relname = 'vestat'; + +RESET track_vacuum_statistics; +DROP TABLE vestat CASCADE; + +SHOW track_vacuum_statistics; -- must be on + +-- ensure pending stats are flushed +SELECT pg_stat_force_next_flush(); + +--SET stats_fetch_consistency = snapshot; +CREATE TABLE vestat (x int) WITH (autovacuum_enabled = off, fillfactor = 10); +INSERT INTO vestat SELECT x FROM generate_series(1,:sample_size) as x; +ANALYZE vestat; + +SELECT oid AS roid from pg_class where relname = 'vestat' \gset + +DELETE FROM vestat WHERE x % 2 = 0; +-- Before the first vacuum execution extended stats view is empty. +SELECT vt.relname,vm_new_frozen_pages,tuples_deleted,relpages,pages_scanned,pages_removed +FROM pg_stat_vacuum_tables vt, pg_class c +WHERE vt.relname = 'vestat' AND vt.relid = c.oid; +SELECT relpages AS rp +FROM pg_class c +WHERE relname = 'vestat' \gset + +VACUUM (PARALLEL 0, BUFFER_USAGE_LIMIT 128, INDEX_CLEANUP OFF) vestat; +-- it is necessary to check the wal statistics +CHECKPOINT; + +-- The table and index extended vacuum statistics should show us that +-- vacuum frozed pages and clean up pages, but pages_removed stayed the same +-- because of not full table have cleaned up +SELECT vt.relname,vm_new_frozen_pages > 0 AS vm_new_frozen_pages,tuples_deleted > 0 AS tuples_deleted,relpages-:rp = 0 AS relpages,pages_scanned > 0 AS pages_scanned,pages_removed = 0 AS pages_removed +FROM pg_stat_vacuum_tables vt, pg_class c +WHERE vt.relname = 'vestat' AND vt.relid = c.oid; +SELECT vm_new_frozen_pages AS fp,tuples_deleted AS td,relpages AS rp, pages_scanned AS ps, pages_removed AS pr +FROM pg_stat_vacuum_tables vt, pg_class c +WHERE vt.relname = 'vestat' AND vt.relid = c.oid \gset + +-- Store WAL advances into variables +SELECT wal_records AS hwr,wal_bytes AS hwb,wal_fpi AS hfpi FROM pg_stat_vacuum_tables WHERE relname = 'vestat' \gset + +-- Look into WAL records deltas. +SELECT wal_records > 0 AS dWR, wal_bytes > 0 AS dWB +FROM pg_stat_vacuum_tables WHERE relname = 'vestat'; + +DELETE FROM vestat;; +VACUUM (PARALLEL 0, BUFFER_USAGE_LIMIT 128, INDEX_CLEANUP OFF) vestat; +-- it is necessary to check the wal statistics +CHECKPOINT; + +-- pages_removed must be increased +SELECT vt.relname,vm_new_frozen_pages-:fp > 0 AS vm_new_frozen_pages,tuples_deleted-:td > 0 AS tuples_deleted,relpages -:rp = 0 AS relpages,pages_scanned-:ps > 0 AS pages_scanned,pages_removed-:pr > 0 AS pages_removed +FROM pg_stat_vacuum_tables vt, pg_class c +WHERE vt.relname = 'vestat' AND vt.relid = c.oid; +SELECT vm_new_frozen_pages AS fp,tuples_deleted AS td,relpages AS rp, pages_scanned AS ps, pages_removed AS pr +FROM pg_stat_vacuum_tables vt, pg_class c +WHERE vt.relname = 'vestat' AND vt.relid = c.oid \gset + +-- Store WAL advances into variables +SELECT wal_records-:hwr AS dwr, wal_bytes-:hwb AS dwb, wal_fpi-:hfpi AS dfpi +FROM pg_stat_vacuum_tables WHERE relname = 'vestat' \gset + +-- WAL advance should be detected. +SELECT :dwr > 0 AS dWR, :dwb > 0 AS dWB; + +-- Store WAL advances into variables +SELECT wal_records AS hwr,wal_bytes AS hwb,wal_fpi AS hfpi FROM pg_stat_vacuum_tables WHERE relname = 'vestat' \gset + +INSERT INTO vestat SELECT x FROM generate_series(1,:sample_size) as x; +DELETE FROM vestat WHERE x % 2 = 0; +-- VACUUM FULL doesn't report to stat collector. So, no any advancements of statistics +-- are detected here. +VACUUM FULL vestat; +-- It is necessary to check the wal statistics +CHECKPOINT; + +-- Store WAL advances into variables +SELECT wal_records-:hwr AS dwr2, wal_bytes-:hwb AS dwb2, wal_fpi-:hfpi AS dfpi2 +FROM pg_stat_vacuum_tables WHERE relname = 'vestat' \gset + +-- WAL and other statistics advance should not be detected. +SELECT :dwr2=0 AS dWR, :dfpi2=0 AS dFPI, :dwb2=0 AS dWB; + +SELECT vt.relname,vm_new_frozen_pages-:fp = 0 AS vm_new_frozen_pages,tuples_deleted-:td = 0 AS tuples_deleted,relpages -:rp < 0 AS relpages,pages_scanned-:ps = 0 AS pages_scanned,pages_removed-:pr = 0 AS pages_removed +FROM pg_stat_vacuum_tables vt, pg_class c +WHERE vt.relname = 'vestat' AND vt.relid = c.oid; +SELECT vm_new_frozen_pages AS fp,tuples_deleted AS td,relpages AS rp, pages_scanned AS ps,pages_removed AS pr +FROM pg_stat_vacuum_tables vt, pg_class c +WHERE vt.relname = 'vestat' AND vt.relid = c.oid \gset + +-- Store WAL advances into variables +SELECT wal_records AS hwr,wal_bytes AS hwb,wal_fpi AS hfpi FROM pg_stat_vacuum_tables WHERE relname = 'vestat' \gset + +DELETE FROM vestat; +TRUNCATE vestat; +VACUUM (PARALLEL 0, BUFFER_USAGE_LIMIT 128, INDEX_CLEANUP OFF) vestat; +-- it is necessary to check the wal statistics +CHECKPOINT; + +-- Store WAL advances into variables after removing all tuples from the table +SELECT wal_records-:hwr AS dwr3, wal_bytes-:hwb AS dwb3, wal_fpi-:hfpi AS dfpi3 +FROM pg_stat_vacuum_tables WHERE relname = 'vestat' \gset + +--There are nothing changed +SELECT :dwr3>0 AS dWR, :dfpi3=0 AS dFPI, :dwb3>0 AS dWB; + +-- +-- Now, the table and index is compressed into zero number of pages. Check it +-- in vacuum extended statistics. +-- The vm_new_frozen_pages, pages_scanned values shouldn't be changed +-- +SELECT vt.relname,vm_new_frozen_pages-:fp = 0 AS vm_new_frozen_pages,tuples_deleted-:td = 0 AS tuples_deleted,relpages -:rp = 0 AS relpages,pages_scanned-:ps = 0 AS pages_scanned,pages_removed-:pr = 0 AS pages_removed +FROM pg_stat_vacuum_tables vt, pg_class c +WHERE vt.relname = 'vestat' AND vt.relid = c.oid; + +INSERT INTO vestat SELECT x FROM generate_series(1,:sample_size) as x; +ANALYZE vestat; + +-- must be empty +SELECT vm_new_frozen_pages, vm_new_visible_pages, rev_all_frozen_pages,rev_all_visible_pages,vm_new_visible_frozen_pages +FROM pg_stat_vacuum_tables, pg_stat_all_tables WHERE pg_stat_vacuum_tables.relname = 'vestat' and pg_stat_vacuum_tables.relid = pg_stat_all_tables.relid; + +VACUUM (PARALLEL 0, BUFFER_USAGE_LIMIT 128) vestat; + +-- backend defreezed pages +SELECT vm_new_frozen_pages > 0 AS vm_new_frozen_pages,vm_new_visible_pages > 0 AS vm_new_visible_pages,vm_new_visible_frozen_pages > 0 AS vm_new_visible_frozen_pages,rev_all_frozen_pages = 0 AS rev_all_frozen_pages,rev_all_visible_pages = 0 AS rev_all_visible_pages +FROM pg_stat_vacuum_tables, pg_stat_all_tables WHERE pg_stat_vacuum_tables.relname = 'vestat' and pg_stat_vacuum_tables.relid = pg_stat_all_tables.relid; +SELECT vm_new_frozen_pages AS pf, vm_new_visible_pages AS pv,vm_new_visible_frozen_pages AS pvf, rev_all_frozen_pages AS hafp,rev_all_visible_pages AS havp +FROM pg_stat_vacuum_tables, pg_stat_all_tables WHERE pg_stat_vacuum_tables.relname = 'vestat' and pg_stat_vacuum_tables.relid = pg_stat_all_tables.relid \gset + +UPDATE vestat SET x = x + 1001; +VACUUM (PARALLEL 0, BUFFER_USAGE_LIMIT 128) vestat; + +SELECT vm_new_frozen_pages > :pf AS vm_new_frozen_pages,vm_new_visible_pages > :pv AS vm_new_visible_pages,vm_new_visible_frozen_pages > :pvf AS vm_new_visible_frozen_pages,rev_all_frozen_pages > :hafp AS rev_all_frozen_pages,rev_all_visible_pages > :havp AS rev_all_visible_pages +FROM pg_stat_vacuum_tables, pg_stat_all_tables WHERE pg_stat_vacuum_tables.relname = 'vestat' and pg_stat_vacuum_tables.relid = pg_stat_all_tables.relid; +SELECT vm_new_frozen_pages AS pf, vm_new_visible_pages AS pv, vm_new_visible_frozen_pages AS pvf, rev_all_frozen_pages AS hafp,rev_all_visible_pages AS havp +FROM pg_stat_vacuum_tables, pg_stat_all_tables WHERE pg_stat_vacuum_tables.relname = 'vestat' and pg_stat_vacuum_tables.relid = pg_stat_all_tables.relid \gset + +VACUUM (PARALLEL 0, BUFFER_USAGE_LIMIT 128) vestat; + +-- vacuum freezed pages +SELECT vm_new_frozen_pages = :pf AS vm_new_frozen_pages,vm_new_visible_pages = :pv AS vm_new_visible_pages,vm_new_visible_frozen_pages = :pvf AS vm_new_visible_frozen_pages, rev_all_frozen_pages = :hafp AS rev_all_frozen_pages,rev_all_visible_pages = :havp AS rev_all_visible_pages +FROM pg_stat_vacuum_tables, pg_stat_all_tables WHERE pg_stat_vacuum_tables.relname = 'vestat' and pg_stat_vacuum_tables.relid = pg_stat_all_tables.relid; + +DROP TABLE vestat CASCADE; -- 2.34.1