From 9b2b6ab2ae56d975f823db830c43a4520771662d Mon Sep 17 00:00:00 2001 From: Alena Rybakina Date: Tue, 8 Oct 2024 19:13:05 +0300 Subject: [PATCH 3/4] Machinery for grabbing an extended vacuum statistics on databases. It transmits vacuum statistical information about each table and accumulates it for the database which the table belonged. --- src/backend/catalog/system_views.sql | 28 +++++++ src/backend/utils/activity/pgstat.c | 2 + src/backend/utils/activity/pgstat_database.c | 1 + src/backend/utils/activity/pgstat_relation.c | 16 ++++ src/backend/utils/adt/pgstatfuncs.c | 76 +++++++++++++++++- src/include/catalog/pg_proc.dat | 11 ++- src/include/pgstat.h | 3 +- src/test/regress/expected/rules.out | 18 +++++ ...ut => vacuum_tables_and_db_statistics.out} | 78 +++++++++++++++++++ src/test/regress/parallel_schedule | 2 +- ...ql => vacuum_tables_and_db_statistics.sql} | 66 +++++++++++++++- 11 files changed, 295 insertions(+), 6 deletions(-) rename src/test/regress/expected/{vacuum_tables_statistics.out => vacuum_tables_and_db_statistics.out} (77%) rename src/test/regress/sql/{vacuum_tables_statistics.sql => vacuum_tables_and_db_statistics.sql} (78%) diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index f5e4e1fbaa5..b63c1804b41 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1474,3 +1474,31 @@ WHERE rel.oid = stats.relid AND ns.oid = rel.relnamespace AND rel.relkind = 'i'; + +CREATE VIEW pg_stat_vacuum_database AS +SELECT + db.oid as dboid, + db.datname AS dbname, + + stats.db_blks_read, + stats.db_blks_hit, + stats.total_blks_dirtied, + stats.total_blks_written, + + stats.wal_records, + stats.wal_fpi, + stats.wal_bytes, + + stats.blk_read_time, + stats.blk_write_time, + + stats.delay_time, + stats.system_time, + stats.user_time, + stats.total_time, + + stats.interrupts +FROM + pg_database db LEFT JOIN pg_stat_vacuum_database(db.oid) stats +ON + db.oid = stats.dboid; \ No newline at end of file diff --git a/src/backend/utils/activity/pgstat.c b/src/backend/utils/activity/pgstat.c index 843617eba25..21b29804620 100644 --- a/src/backend/utils/activity/pgstat.c +++ b/src/backend/utils/activity/pgstat.c @@ -1124,6 +1124,8 @@ pgstat_update_snapshot(PgStat_Kind kind) pgstat_fetch_consistency = PGSTAT_FETCH_CONSISTENCY_SNAPSHOT; if (kind == PGSTAT_KIND_RELATION) pgstat_build_snapshot(PGSTAT_KIND_RELATION); + else if (kind == PGSTAT_KIND_DATABASE) + pgstat_build_snapshot(PGSTAT_KIND_DATABASE); } PG_FINALLY(); { diff --git a/src/backend/utils/activity/pgstat_database.c b/src/backend/utils/activity/pgstat_database.c index 29bc0909748..a060d1a4042 100644 --- a/src/backend/utils/activity/pgstat_database.c +++ b/src/backend/utils/activity/pgstat_database.c @@ -430,6 +430,7 @@ pgstat_database_flush_cb(PgStat_EntryRef *entry_ref, bool nowait) pgstat_unlock_entry(entry_ref); memset(pendingent, 0, sizeof(*pendingent)); + memset(&(pendingent)->vacuum_ext, 0, sizeof(ExtVacReport)); return true; } diff --git a/src/backend/utils/activity/pgstat_relation.c b/src/backend/utils/activity/pgstat_relation.c index 5c95363c04a..725e26423f2 100644 --- a/src/backend/utils/activity/pgstat_relation.c +++ b/src/backend/utils/activity/pgstat_relation.c @@ -219,6 +219,7 @@ pgstat_report_vacuum_error(Oid tableoid, ExtVacReportType m_type) PgStatShared_Relation *shtabentry; PgStat_StatTabEntry *tabentry; Oid dboid = MyDatabaseId; + PgStat_StatDBEntry *dbentry; /* pending database entry */ if (!pgstat_track_counts) return; @@ -232,6 +233,10 @@ pgstat_report_vacuum_error(Oid tableoid, ExtVacReportType m_type) tabentry->vacuum_ext.interrupts++; tabentry->vacuum_ext.type = m_type; pgstat_unlock_entry(entry_ref); + + dbentry = pgstat_prep_database_pending(dboid); + dbentry->vacuum_ext.interrupts++; + dbentry->vacuum_ext.type = m_type; } /* @@ -245,6 +250,7 @@ pgstat_report_vacuum(Oid tableoid, bool shared, PgStat_EntryRef *entry_ref; PgStatShared_Relation *shtabentry; PgStat_StatTabEntry *tabentry; + PgStatShared_Database *dbentry; Oid dboid = (shared ? InvalidOid : MyDatabaseId); TimestampTz ts; @@ -298,6 +304,16 @@ pgstat_report_vacuum(Oid tableoid, bool shared, * VACUUM command has processed all tables and committed. */ pgstat_flush_io(false); + if (dboid != InvalidOid) + { + entry_ref = pgstat_get_entry_ref_locked(PGSTAT_KIND_DATABASE, + dboid, InvalidOid, false); + dbentry = (PgStatShared_Database *) entry_ref->shared_stats; + + pgstat_accumulate_extvac_stats(&dbentry->stats.vacuum_ext, params, false); + pgstat_unlock_entry(entry_ref); + } + } /* diff --git a/src/backend/utils/adt/pgstatfuncs.c b/src/backend/utils/adt/pgstatfuncs.c index e698d637860..4d1c099b37e 100644 --- a/src/backend/utils/adt/pgstatfuncs.c +++ b/src/backend/utils/adt/pgstatfuncs.c @@ -2102,8 +2102,49 @@ pg_stat_have_stats(PG_FUNCTION_ARGS) #define EXTVACHEAPSTAT_COLUMNS 27 #define EXTVACIDXSTAT_COLUMNS 19 +#define EXTVACDBSTAT_COLUMNS 15 #define EXTVACSTAT_COLUMNS Max(EXTVACHEAPSTAT_COLUMNS, EXTVACIDXSTAT_COLUMNS) +static void +tuplestore_put_for_database(Oid dbid, ReturnSetInfo *rsinfo, + PgStatShared_Database *dbentry) +{ + Datum values[EXTVACDBSTAT_COLUMNS]; + bool nulls[EXTVACDBSTAT_COLUMNS]; + char buf[256]; + int i = 0; + + memset(nulls, 0, EXTVACDBSTAT_COLUMNS * sizeof(bool)); + + values[i++] = ObjectIdGetDatum(dbid); + + values[i++] = Int64GetDatum(dbentry->stats.vacuum_ext.total_blks_read); + values[i++] = Int64GetDatum(dbentry->stats.vacuum_ext.total_blks_hit); + values[i++] = Int64GetDatum(dbentry->stats.vacuum_ext.total_blks_dirtied); + values[i++] = Int64GetDatum(dbentry->stats.vacuum_ext.total_blks_written); + + values[i++] = Int64GetDatum(dbentry->stats.vacuum_ext.wal_records); + values[i++] = Int64GetDatum(dbentry->stats.vacuum_ext.wal_fpi); + + /* Convert to numeric, like pg_stat_statements */ + snprintf(buf, sizeof buf, UINT64_FORMAT, dbentry->stats.vacuum_ext.wal_bytes); + values[i++] = DirectFunctionCall3(numeric_in, + CStringGetDatum(buf), + ObjectIdGetDatum(0), + Int32GetDatum(-1)); + + values[i++] = Float8GetDatum(dbentry->stats.vacuum_ext.blk_read_time); + values[i++] = Float8GetDatum(dbentry->stats.vacuum_ext.blk_write_time); + values[i++] = Float8GetDatum(dbentry->stats.vacuum_ext.delay_time); + values[i++] = Float8GetDatum(dbentry->stats.vacuum_ext.system_time); + values[i++] = Float8GetDatum(dbentry->stats.vacuum_ext.user_time); + values[i++] = Float8GetDatum(dbentry->stats.vacuum_ext.total_time); + values[i++] = Int32GetDatum(dbentry->stats.vacuum_ext.interrupts); + + Assert(i == rsinfo->setDesc->natts); + tuplestore_putvalues(rsinfo->setResult, rsinfo->setDesc, values, nulls); +} + static void tuplestore_put_for_relation(Oid relid, ReturnSetInfo *rsinfo, PgStat_StatTabEntry *tabentry) @@ -2218,10 +2259,30 @@ pg_stats_vacuum(FunctionCallInfo fcinfo, ExtVacReportType type, int ncolumns) tabentry = (PgStat_StatTabEntry *) entry->data; if (tabentry != NULL && tabentry->vacuum_ext.type == type) - tuplestore_put_for_relation(entry->key.objoid, rsinfo, tabentry); + tuplestore_put_for_relation(entry->key.objid, rsinfo, tabentry); } } } + else if (type == PGSTAT_EXTVAC_DB) + { + PgStatShared_Database *dbentry; + PgStat_EntryRef *entry_ref; + Oid dbid = PG_GETARG_OID(0); + + if (OidIsValid(dbid)) + { + entry_ref = pgstat_get_entry_ref_locked(PGSTAT_KIND_DATABASE, + dbid, InvalidOid, false); + dbentry = (PgStatShared_Database *) entry_ref->shared_stats; + + if (dbentry == NULL) + /* Table doesn't exist or isn't a heap relation */ + return; + + tuplestore_put_for_database(dbid, rsinfo, dbentry); + pgstat_unlock_entry(entry_ref); + } + } } /* @@ -2230,7 +2291,7 @@ pg_stats_vacuum(FunctionCallInfo fcinfo, ExtVacReportType type, int ncolumns) Datum pg_stat_vacuum_tables(PG_FUNCTION_ARGS) { - pg_stats_vacuum(fcinfo, EXTVACHEAPSTAT_COLUMNS); + pg_stats_vacuum(fcinfo, PGSTAT_EXTVAC_HEAP, EXTVACHEAPSTAT_COLUMNS); PG_RETURN_VOID(); } @@ -2243,5 +2304,16 @@ pg_stat_vacuum_indexes(PG_FUNCTION_ARGS) { pg_stats_vacuum(fcinfo, PGSTAT_EXTVAC_INDEX, EXTVACIDXSTAT_COLUMNS); + PG_RETURN_VOID(); +} + +/* + * Get the vacuum statistics for the database. + */ +Datum +pg_stat_vacuum_database(PG_FUNCTION_ARGS) +{ + pg_stats_vacuum(fcinfo, PGSTAT_EXTVAC_DB, EXTVACDBSTAT_COLUMNS); + PG_RETURN_VOID(); } \ No newline at end of file diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index 9723551a73a..936713fe5c1 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -12346,5 +12346,14 @@ proallargtypes => '{oid,oid,int8,int8,int8,int8,int8,int8,int8,int8,int8,int8,numeric,float8,float8,float8,float8,float8,float8,int4}', proargmodes => '{i,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o}', proargnames => '{reloid,relid,total_blks_read,total_blks_hit,total_blks_dirtied,total_blks_written,rel_blks_read,rel_blks_hit,pages_deleted,tuples_deleted,wal_records,wal_fpi,wal_bytes,blk_read_time,blk_write_time,delay_time,system_time,user_time,total_time,interrupts}', - prosrc => 'pg_stat_vacuum_indexes' } + prosrc => 'pg_stat_vacuum_indexes' }, +{ oid => '8003', + descr => 'pg_stat_vacuum_database return stats values', + proname => 'pg_stat_vacuum_database', prorows => 1000, provolatile => 's', prorettype => 'record',proisstrict => 'f', + proretset => 't', + proargtypes => 'oid', + proallargtypes => '{oid,oid,int8,int8,int8,int8,int8,int8,numeric,float8,float8,float8,float8,float8,float8,int4}', + proargmodes => '{i,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o}', + proargnames => '{dbid,dboid,db_blks_read,db_blks_hit,total_blks_dirtied,total_blks_written,wal_records,wal_fpi,wal_bytes,blk_read_time,blk_write_time,delay_time,system_time,user_time,total_time,interrupts}', + prosrc => 'pg_stat_vacuum_database' }, ] diff --git a/src/include/pgstat.h b/src/include/pgstat.h index b784bcc3efe..c6d663c1c48 100644 --- a/src/include/pgstat.h +++ b/src/include/pgstat.h @@ -174,7 +174,8 @@ typedef enum ExtVacReportType { PGSTAT_EXTVAC_INVALID = 0, PGSTAT_EXTVAC_HEAP = 1, - PGSTAT_EXTVAC_INDEX = 2 + PGSTAT_EXTVAC_INDEX = 2, + PGSTAT_EXTVAC_DB = 3, } ExtVacReportType; /* ---------- diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out index e3290da748d..8359cf3e984 100644 --- a/src/test/regress/expected/rules.out +++ b/src/test/regress/expected/rules.out @@ -2237,6 +2237,24 @@ pg_stat_user_tables| SELECT relid, autoanalyze_count FROM pg_stat_all_tables WHERE ((schemaname <> ALL (ARRAY['pg_catalog'::name, 'information_schema'::name])) AND (schemaname !~ '^pg_toast'::text)); +pg_stat_vacuum_database| SELECT db.oid AS dboid, + db.datname AS dbname, + stats.db_blks_read, + stats.db_blks_hit, + stats.total_blks_dirtied, + stats.total_blks_written, + stats.wal_records, + stats.wal_fpi, + stats.wal_bytes, + stats.blk_read_time, + stats.blk_write_time, + stats.delay_time, + stats.system_time, + stats.user_time, + stats.total_time, + stats.interrupts + FROM (pg_database db + LEFT JOIN LATERAL pg_stat_vacuum_database(db.oid) stats(dboid, db_blks_read, db_blks_hit, total_blks_dirtied, total_blks_written, wal_records, wal_fpi, wal_bytes, blk_read_time, blk_write_time, delay_time, system_time, user_time, total_time, interrupts) ON ((db.oid = stats.dboid))); pg_stat_vacuum_indexes| SELECT rel.oid AS relid, ns.nspname AS schema, rel.relname, diff --git a/src/test/regress/expected/vacuum_tables_statistics.out b/src/test/regress/expected/vacuum_tables_and_db_statistics.out similarity index 77% rename from src/test/regress/expected/vacuum_tables_statistics.out rename to src/test/regress/expected/vacuum_tables_and_db_statistics.out index 86272217e5d..94dd3214349 100644 --- a/src/test/regress/expected/vacuum_tables_statistics.out +++ b/src/test/regress/expected/vacuum_tables_and_db_statistics.out @@ -6,6 +6,9 @@ -- number of frozen and visible pages removed by backend. -- Statistic wal_fpi is not displayed in this test because its behavior is unstable. -- +CREATE DATABASE regression_statistic_vacuum_db; +CREATE DATABASE regression_statistic_vacuum_db1; +\c regression_statistic_vacuum_db; -- conditio sine qua non SHOW track_counts; -- must be on track_counts @@ -205,4 +208,79 @@ SELECT min(relid) FROM pg_stat_vacuum_tables(0) where relid > 0; 1213 (1 row) +-- Now check vacuum statistics for current database +SELECT dbname, + db_blks_hit > 0 AS db_blks_hit, + total_blks_dirtied > 0 AS total_blks_dirtied, + total_blks_written > 0 AS total_blks_written, + wal_records > 0 AS wal_records, + wal_fpi > 0 AS wal_fpi, + wal_bytes > 0 AS wal_bytes, + user_time > 0 AS user_time, + total_time > 0 AS total_time +FROM +pg_stat_vacuum_database +WHERE dbname = current_database(); + dbname | db_blks_hit | total_blks_dirtied | total_blks_written | wal_records | wal_fpi | wal_bytes | user_time | total_time +--------------------------------+-------------+--------------------+--------------------+-------------+---------+-----------+-----------+------------ + regression_statistic_vacuum_db | t | t | t | t | t | t | t | t +(1 row) + +DROP TABLE vestat CASCADE; +-- ensure pending stats are flushed +SELECT pg_stat_force_next_flush(); + pg_stat_force_next_flush +-------------------------- + +(1 row) + +CREATE TABLE vestat (x int) WITH (autovacuum_enabled = off, fillfactor = 10); +INSERT INTO vestat SELECT x FROM generate_series(1,:sample_size) as x; +ANALYZE vestat; +UPDATE vestat SET x = 10001; +VACUUM (PARALLEL 0, BUFFER_USAGE_LIMIT 128) vestat; +\c regression_statistic_vacuum_db1; +-- Now check vacuum statistics for postgres database from another database +SELECT dbname, + db_blks_hit > 0 AS db_blks_hit, + total_blks_dirtied > 0 AS total_blks_dirtied, + total_blks_written > 0 AS total_blks_written, + wal_records > 0 AS wal_records, + wal_fpi > 0 AS wal_fpi, + wal_bytes > 0 AS wal_bytes, + user_time > 0 AS user_time, + total_time > 0 AS total_time +FROM +pg_stat_vacuum_database +WHERE dbname = 'regression_statistic_vacuum_db'; + dbname | db_blks_hit | total_blks_dirtied | total_blks_written | wal_records | wal_fpi | wal_bytes | user_time | total_time +--------------------------------+-------------+--------------------+--------------------+-------------+---------+-----------+-----------+------------ + regression_statistic_vacuum_db | t | t | t | t | t | t | t | t +(1 row) + +\c regression_statistic_vacuum_db +RESET vacuum_freeze_min_age; +RESET vacuum_freeze_table_age; DROP TABLE vestat CASCADE; +\c regression_statistic_vacuum_db1; +SELECT count(*) +FROM pg_database d +CROSS JOIN pg_stat_vacuum_tables(0) +WHERE oid = 0; -- must be 0 + count +------- + 0 +(1 row) + +SELECT count(*) +FROM pg_database d +CROSS JOIN pg_stat_vacuum_database(0) +WHERE oid = 0; -- must be 0 + count +------- + 0 +(1 row) + +\c postgres +DROP DATABASE regression_statistic_vacuum_db1; +DROP DATABASE regression_statistic_vacuum_db; diff --git a/src/test/regress/parallel_schedule b/src/test/regress/parallel_schedule index 34fd3e4e674..e999232d429 100644 --- a/src/test/regress/parallel_schedule +++ b/src/test/regress/parallel_schedule @@ -141,4 +141,4 @@ test: tablespace # Check vacuum statistics # ---------- test: vacuum_index_statistics -test: vacuum_tables_statistics \ No newline at end of file +test: vacuum_tables_and_db_statistics \ No newline at end of file diff --git a/src/test/regress/sql/vacuum_tables_statistics.sql b/src/test/regress/sql/vacuum_tables_and_db_statistics.sql similarity index 78% rename from src/test/regress/sql/vacuum_tables_statistics.sql rename to src/test/regress/sql/vacuum_tables_and_db_statistics.sql index bc8d051aefa..af1281b3b63 100644 --- a/src/test/regress/sql/vacuum_tables_statistics.sql +++ b/src/test/regress/sql/vacuum_tables_and_db_statistics.sql @@ -7,6 +7,10 @@ -- Statistic wal_fpi is not displayed in this test because its behavior is unstable. -- +CREATE DATABASE regression_statistic_vacuum_db; +CREATE DATABASE regression_statistic_vacuum_db1; +\c regression_statistic_vacuum_db; + -- conditio sine qua non SHOW track_counts; -- must be on -- not enabled by default, but we want to test it... @@ -157,4 +161,64 @@ FROM pg_stat_vacuum_tables WHERE relname = 'vestat'; SELECT min(relid) FROM pg_stat_vacuum_tables(0) where relid > 0; -DROP TABLE vestat CASCADE; \ No newline at end of file +-- Now check vacuum statistics for current database +SELECT dbname, + db_blks_hit > 0 AS db_blks_hit, + total_blks_dirtied > 0 AS total_blks_dirtied, + total_blks_written > 0 AS total_blks_written, + wal_records > 0 AS wal_records, + wal_fpi > 0 AS wal_fpi, + wal_bytes > 0 AS wal_bytes, + user_time > 0 AS user_time, + total_time > 0 AS total_time +FROM +pg_stat_vacuum_database +WHERE dbname = current_database(); + +DROP TABLE vestat CASCADE; + +-- ensure pending stats are flushed +SELECT pg_stat_force_next_flush(); + +CREATE TABLE vestat (x int) WITH (autovacuum_enabled = off, fillfactor = 10); +INSERT INTO vestat SELECT x FROM generate_series(1,:sample_size) as x; +ANALYZE vestat; +UPDATE vestat SET x = 10001; +VACUUM (PARALLEL 0, BUFFER_USAGE_LIMIT 128) vestat; + +\c regression_statistic_vacuum_db1; + +-- Now check vacuum statistics for postgres database from another database +SELECT dbname, + db_blks_hit > 0 AS db_blks_hit, + total_blks_dirtied > 0 AS total_blks_dirtied, + total_blks_written > 0 AS total_blks_written, + wal_records > 0 AS wal_records, + wal_fpi > 0 AS wal_fpi, + wal_bytes > 0 AS wal_bytes, + user_time > 0 AS user_time, + total_time > 0 AS total_time +FROM +pg_stat_vacuum_database +WHERE dbname = 'regression_statistic_vacuum_db'; + +\c regression_statistic_vacuum_db + +RESET vacuum_freeze_min_age; +RESET vacuum_freeze_table_age; +DROP TABLE vestat CASCADE; + +\c regression_statistic_vacuum_db1; +SELECT count(*) +FROM pg_database d +CROSS JOIN pg_stat_vacuum_tables(0) +WHERE oid = 0; -- must be 0 + +SELECT count(*) +FROM pg_database d +CROSS JOIN pg_stat_vacuum_database(0) +WHERE oid = 0; -- must be 0 + +\c postgres +DROP DATABASE regression_statistic_vacuum_db1; +DROP DATABASE regression_statistic_vacuum_db; -- 2.34.1