agora inbox for [email protected]help / color / mirror / Atom feed
[PATCH] Add a function to reset the cumulative statistics 25+ messages / 4 participants [nested] [flat]
* [PATCH] Add a function to reset the cumulative statistics @ 2019-02-14 13:37 Pierre <[email protected]> 0 siblings, 0 replies; 25+ messages in thread From: Pierre @ 2019-02-14 13:37 UTC (permalink / raw) pg_stat_statements has two parts : raw statistics that are simple 'stable' counters, and computed statistics (averages, min time, max time...) When using pg_stat_statements to find and fix performance issues, being able to reset the computed statistics can help track the issue and the impact of the fixes. This would also make it possible for tools like powa to collect these statistics too and track them over time by resetting them after each collection. --- .../pg_stat_statements--1.7--1.8.sql | 12 +++++ .../pg_stat_statements/pg_stat_statements.c | 51 +++++++++++++++++-- .../pg_stat_statements.control | 2 +- doc/src/sgml/pgstatstatements.sgml | 25 +++++++++ 4 files changed, 86 insertions(+), 4 deletions(-) create mode 100644 contrib/pg_stat_statements/pg_stat_statements--1.7--1.8.sql diff --git a/contrib/pg_stat_statements/pg_stat_statements--1.7--1.8.sql b/contrib/pg_stat_statements/pg_stat_statements--1.7--1.8.sql new file mode 100644 index 0000000000..351187e0a1 --- /dev/null +++ b/contrib/pg_stat_statements/pg_stat_statements--1.7--1.8.sql @@ -0,0 +1,12 @@ +/* contrib/pg_stat_statements/pg_stat_statements--1.7--1.8.sql */ + +-- complain if script is sourced in psql, rather than via ALTER EXTENSION +\echo Use "ALTER EXTENSION pg_stat_statements UPDATE TO '1.8'" to load this file. \quit + +CREATE FUNCTION pg_stat_statements_reset_computed_values() +RETURNS void +AS 'MODULE_PATHNAME' +LANGUAGE C PARALLEL SAFE; + +-- Don't want this to be available to non-superusers. +REVOKE ALL ON FUNCTION pg_stat_statements_reset_computed_values() FROM PUBLIC; diff --git a/contrib/pg_stat_statements/pg_stat_statements.c b/contrib/pg_stat_statements/pg_stat_statements.c index 221b47298c..fa6bdf57bd 100644 --- a/contrib/pg_stat_statements/pg_stat_statements.c +++ b/contrib/pg_stat_statements/pg_stat_statements.c @@ -150,6 +150,7 @@ typedef struct Counters double max_time; /* maximum execution time in msec */ double mean_time; /* mean execution time in msec */ double sum_var_time; /* sum of variances in execution time in msec */ + int64 computed_calls; /* # of times executed considered for the previous computed values */ int64 rows; /* total # of retrieved or affected rows */ int64 shared_blks_hit; /* # of shared buffer hits */ int64 shared_blks_read; /* # of shared disk blocks read */ @@ -289,6 +290,7 @@ static bool pgss_save; /* whether to save stats across shutdown */ void _PG_init(void); void _PG_fini(void); +PG_FUNCTION_INFO_V1(pg_stat_statements_reset_computed_values); PG_FUNCTION_INFO_V1(pg_stat_statements_reset); PG_FUNCTION_INFO_V1(pg_stat_statements_reset_1_7); PG_FUNCTION_INFO_V1(pg_stat_statements_1_2); @@ -1252,8 +1254,9 @@ pgss_store(const char *query, uint64 queryId, e->counters.usage = USAGE_INIT; e->counters.calls += 1; + e->counters.computed_calls += 1; e->counters.total_time += total_time; - if (e->counters.calls == 1) + if (e->counters.computed_calls == 1) { e->counters.min_time = total_time; e->counters.max_time = total_time; @@ -1268,7 +1271,7 @@ pgss_store(const char *query, uint64 queryId, double old_mean = e->counters.mean_time; e->counters.mean_time += - (total_time - old_mean) / e->counters.calls; + (total_time - old_mean) / e->counters.computed_calls; e->counters.sum_var_time += (total_time - old_mean) * (total_time - e->counters.mean_time); @@ -1324,7 +1327,7 @@ pg_stat_statements_reset_1_7(PG_FUNCTION_ARGS) } /* - * Reset statement statistics. + * Reset all statement statistics. */ Datum pg_stat_statements_reset(PG_FUNCTION_ARGS) @@ -1334,6 +1337,48 @@ pg_stat_statements_reset(PG_FUNCTION_ARGS) PG_RETURN_VOID(); } +/* + * Reset computed statistics from all statements. + */ +Datum +pg_stat_statements_reset_computed_values(PG_FUNCTION_ARGS) +{ + if (!pgss || !pgss_hash) + ereport(ERROR, + (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), + errmsg("pg_stat_statements must be loaded via shared_preload_libraries"))); + entry_reset_computed(); + PG_RETURN_VOID(); +} + + +/* + * Reset computed statement statistics - inner function. + */ +static void +entry_reset_computed(void) +{ + HASH_SEQ_STATUS hash_seq; + pgssEntry *entry; + + /* Lookup the hash table entry with shared lock. */ + LWLockAcquire(pgss->lock, LW_SHARED); + + hash_seq_init(&hash_seq, pgss_hash); + while ((entry = hash_seq_search(&hash_seq)) != NULL) + { + SpinLockAcquire(&entry->mutex); + entry->counters.computed_calls = 0; + entry->counters.min_time = 0; + entry->counters.max_time = 0; + entry->counters.mean_time = 0; + entry->counters.sum_var_time = 0; + SpinLockRelease(&entry->mutex); + } + + LWLockRelease(pgss->lock); +} + /* Number of output arguments (columns) for various API versions */ #define PG_STAT_STATEMENTS_COLS_V1_0 14 #define PG_STAT_STATEMENTS_COLS_V1_1 18 diff --git a/contrib/pg_stat_statements/pg_stat_statements.control b/contrib/pg_stat_statements/pg_stat_statements.control index 14cb422354..7fb20df886 100644 --- a/contrib/pg_stat_statements/pg_stat_statements.control +++ b/contrib/pg_stat_statements/pg_stat_statements.control @@ -1,5 +1,5 @@ # pg_stat_statements extension comment = 'track execution statistics of all SQL statements executed' -default_version = '1.7' +default_version = '1.8' module_pathname = '$libdir/pg_stat_statements' relocatable = true diff --git a/doc/src/sgml/pgstatstatements.sgml b/doc/src/sgml/pgstatstatements.sgml index 26bb82da4a..6958b233f3 100644 --- a/doc/src/sgml/pgstatstatements.sgml +++ b/doc/src/sgml/pgstatstatements.sgml @@ -387,6 +387,31 @@ </para> </listitem> </varlistentry> + + <varlistentry> + <term> + <function>pg_stat_statements_reset_computed_values() returns void</function> + <indexterm> + <primary>pg_stat_statements_reset_computed_values</primary> + </indexterm> + </term> + + <listitem> + <para> + <function>pg_stat_statements_reset_computed_values</function> discards the + computed part of statistics gathered so far by <filename>pg_stat_statements</filename>, + aka the min_time, max_time, mean_time and sum_var_time fields. These fields, + unlike the others, are not plain cumulative values and instead the results of + an internal computation. When doing statistics on statements execution or + graphing them in an external tool, it can thus be usefull to be able to reset + only these instead of resetting the whole statistics. By default, this + function can only be executed by superusers. Access may be granted to others + using <command>GRANT</command>. + </para> + </listitem> + </varlistentry> + + </variablelist> </sect2> -- 2.23.0.rc1 --nextPart2910196.vPqlLT9RqA-- ^ permalink raw reply [nested|flat] 25+ messages in thread
* [PATCH v7 9/9] Add explain(REGRESS)... @ 2020-02-23 03:17 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 25+ messages in thread From: Justin Pryzby @ 2020-02-23 03:17 UTC (permalink / raw) shorthand for: costs off, timing off, summary off, memory off --- src/backend/commands/explain.c | 24 ++++++++++++++++--- src/test/regress/expected/select.out | 2 +- src/test/regress/expected/select_parallel.out | 2 +- src/test/regress/sql/select.sql | 2 +- src/test/regress/sql/select_parallel.sql | 2 +- 5 files changed, 25 insertions(+), 7 deletions(-) diff --git a/src/backend/commands/explain.c b/src/backend/commands/explain.c index 9c7a243a78..106b0cdcf6 100644 --- a/src/backend/commands/explain.c +++ b/src/backend/commands/explain.c @@ -167,6 +167,8 @@ ExplainQuery(ParseState *pstate, ExplainStmt *stmt, bool timing_set = false; bool summary_set = false; bool machine_set = false; + bool costs_set = false; + bool regress = false; /* Parse options list. */ foreach(lc, stmt->options) @@ -178,11 +180,18 @@ ExplainQuery(ParseState *pstate, ExplainStmt *stmt, else if (strcmp(opt->defname, "verbose") == 0) es->verbose = defGetBoolean(opt); else if (strcmp(opt->defname, "costs") == 0) + { + /* Need to keep track if it explicitly set to ON */ + costs_set = true; es->costs = defGetBoolean(opt); + } else if (strcmp(opt->defname, "buffers") == 0) es->buffers = defGetBoolean(opt); else if (strcmp(opt->defname, "settings") == 0) es->settings = defGetBoolean(opt); + else if (strcmp(opt->defname, "regress") == 0) + /* This is convenience shorthand, only */ + regress = defGetBoolean(opt); else if (strcmp(opt->defname, "timing") == 0) { timing_set = true; @@ -225,13 +234,22 @@ ExplainQuery(ParseState *pstate, ExplainStmt *stmt, parser_errposition(pstate, opt->location))); } + if (regress && (es->timing || es->machine || es->summary || + (costs_set && es->costs))) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("EXPLAIN option REGRESS cannot be specified with any of: TIMING, MEMORY, SUMMARY, COSTS"))); + + /* if the costs option was not set explicitly, set default value */ + es->costs = (costs_set) ? es->costs : es->costs && !regress; + if (es->buffers && !es->analyze) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("EXPLAIN option BUFFERS requires ANALYZE"))); /* if the timing was not set explicitly, set default value */ - es->timing = (timing_set) ? es->timing : es->analyze; + es->timing = (timing_set) ? es->timing : es->analyze && !regress; /* check that timing is used with EXPLAIN ANALYZE */ if (es->timing && !es->analyze) @@ -246,10 +264,10 @@ ExplainQuery(ParseState *pstate, ExplainStmt *stmt, errmsg("EXPLAIN option MEMORY requires ANALYZE"))); /* if the summary was not set explicitly, set default value */ - es->summary = (summary_set) ? es->summary : es->analyze; + es->summary = (summary_set) ? es->summary : es->analyze && !regress; /* if the memory option was not set explicitly, set default value */ - es->machine = (machine_set) ? es->machine : es->analyze; + es->machine = (machine_set) ? es->machine : es->analyze && !regress; /* * Parse analysis was done already, but we still have to run the rule diff --git a/src/test/regress/expected/select.out b/src/test/regress/expected/select.out index c441049f41..2a94111282 100644 --- a/src/test/regress/expected/select.out +++ b/src/test/regress/expected/select.out @@ -753,7 +753,7 @@ select * from onek2 where unique2 = 11 and stringu1 = 'ATAAAA'; (1 row) -- actually run the query with an analyze to use the partial index -explain (costs off, analyze on, timing off, summary off) +explain (analyze, regress) select * from onek2 where unique2 = 11 and stringu1 = 'ATAAAA'; QUERY PLAN ----------------------------------------------------------------- diff --git a/src/test/regress/expected/select_parallel.out b/src/test/regress/expected/select_parallel.out index dcbf4b385f..51036289ba 100644 --- a/src/test/regress/expected/select_parallel.out +++ b/src/test/regress/expected/select_parallel.out @@ -549,7 +549,7 @@ select count(*) from bmscantest where a>1; -- test accumulation of stats for parallel nodes reset enable_seqscan; alter table tenk2 set (parallel_workers = 0); -explain (analyze, timing off, summary off, costs off) +explain (analyze, regress) select count(*) from tenk1, tenk2 where tenk1.hundred > 1 and tenk2.thousand=0; QUERY PLAN diff --git a/src/test/regress/sql/select.sql b/src/test/regress/sql/select.sql index b5929b2eca..1c757fe0bf 100644 --- a/src/test/regress/sql/select.sql +++ b/src/test/regress/sql/select.sql @@ -196,7 +196,7 @@ explain (costs off) select * from onek2 where unique2 = 11 and stringu1 = 'ATAAAA'; select * from onek2 where unique2 = 11 and stringu1 = 'ATAAAA'; -- actually run the query with an analyze to use the partial index -explain (costs off, analyze on, timing off, summary off) +explain (analyze, regress) select * from onek2 where unique2 = 11 and stringu1 = 'ATAAAA'; explain (costs off) select unique2 from onek2 where unique2 = 11 and stringu1 = 'ATAAAA'; diff --git a/src/test/regress/sql/select_parallel.sql b/src/test/regress/sql/select_parallel.sql index 5be2fef2d3..7afa55f345 100644 --- a/src/test/regress/sql/select_parallel.sql +++ b/src/test/regress/sql/select_parallel.sql @@ -217,7 +217,7 @@ select count(*) from bmscantest where a>1; -- test accumulation of stats for parallel nodes reset enable_seqscan; alter table tenk2 set (parallel_workers = 0); -explain (analyze, timing off, summary off, costs off) +explain (analyze, regress) select count(*) from tenk1, tenk2 where tenk1.hundred > 1 and tenk2.thousand=0; alter table tenk2 reset (parallel_workers); -- 2.17.0 --UfEAyuTBtIjiZzX6-- ^ permalink raw reply [nested|flat] 25+ messages in thread
* [PATCH v2 1/2] per backend I/O statistics @ 2024-08-22 15:16 Bertrand Drouvot <[email protected]> 0 siblings, 0 replies; 25+ messages in thread From: Bertrand Drouvot @ 2024-08-22 15:16 UTC (permalink / raw) While pg_stat_io provides cluster-wide I/O statistics, this commit adds a new pg_my_stat_io view to display "my" backend I/O statistics. The KIND_IO stats are still "fixed amount" ones as the maximum number of backend is fixed. The statistics snapshot is made for the global stats (the aggregated ones) and for my backend stats. The snapshot is not build in each backend to copy all the others backends stats, as 1/ there is no use case (there is no need to get others backends I/O statistics while taking care of the stats_fetch_consistency) and 2/ that could be memory expensive depending of the number of max connections. A subsequent commit will add a new pg_stat_get_backend_io() function to be able to retrieve the I/O statistics for a given backend pid (each execution re-fetches counters from shared memory because as stated above there is no snapshots being created in each backend to copy the other backends stats). Bump catalog version needs to be done. --- doc/src/sgml/config.sgml | 4 +- doc/src/sgml/monitoring.sgml | 28 ++++++ src/backend/catalog/system_views.sql | 22 +++++ src/backend/utils/activity/pgstat.c | 11 ++- src/backend/utils/activity/pgstat_io.c | 91 ++++++++++++++--- src/backend/utils/activity/pgstat_shmem.c | 4 +- src/backend/utils/adt/pgstatfuncs.c | 114 +++++++++++++++++++++- src/include/catalog/pg_proc.dat | 9 ++ src/include/pgstat.h | 14 ++- src/include/utils/pgstat_internal.h | 14 ++- src/test/regress/expected/rules.out | 19 ++++ src/test/regress/expected/stats.out | 44 ++++++++- src/test/regress/sql/stats.sql | 25 ++++- 13 files changed, 366 insertions(+), 33 deletions(-) 8.5% doc/src/sgml/ 29.4% src/backend/utils/activity/ 23.9% src/backend/utils/adt/ 4.4% src/include/catalog/ 3.1% src/include/utils/ 3.1% src/include/ 14.4% src/test/regress/expected/ 10.0% src/test/regress/sql/ diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 0aec11f443..2a59b97093 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -8331,7 +8331,9 @@ COPY postgres_log FROM '/full/path/to/logfile.csv' WITH csv; displayed in <link linkend="monitoring-pg-stat-database-view"> <structname>pg_stat_database</structname></link>, <link linkend="monitoring-pg-stat-io-view"> - <structname>pg_stat_io</structname></link>, in the output of + <structname>pg_stat_io</structname></link>, + <link linkend="monitoring-pg-my-stat-io-view"> + <structname>pg_my_stat_io</structname></link>, in the output of <xref linkend="sql-explain"/> when the <literal>BUFFERS</literal> option is used, in the output of <xref linkend="sql-vacuum"/> when the <literal>VERBOSE</literal> option is used, by autovacuum diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml index 933de6fe07..b28ca4e0ca 100644 --- a/doc/src/sgml/monitoring.sgml +++ b/doc/src/sgml/monitoring.sgml @@ -488,6 +488,16 @@ postgres 27093 0.0 0.0 30096 2752 ? Ss 11:34 0:00 postgres: ser </entry> </row> + <row> + <entry><structname>pg_my_stat_io</structname><indexterm><primary>pg_my_stat_io</primary></indexterm></entry> + <entry> + One row for each combination of context and target object containing + my backend I/O statistics. + See <link linkend="monitoring-pg-my-stat-io-view"> + <structname>pg_my_stat_io</structname></link> for details. + </entry> + </row> + <row> <entry><structname>pg_stat_replication_slots</structname><indexterm><primary>pg_stat_replication_slots</primary></indexterm></entry> <entry>One row per replication slot, showing statistics about the @@ -2948,6 +2958,24 @@ description | Waiting for a newly initialized WAL file to reach durable storage + </sect2> + + <sect2 id="monitoring-pg-my-stat-io-view"> + <title><structname>pg_my_stat_io</structname></title> + + <indexterm> + <primary>pg_my_stat_io</primary> + </indexterm> + + <para> + The <structname>pg_my_stat_io</structname> view will contain one row for each + combination of target I/O object and I/O context, showing + my backend I/O statistics. Combinations which do not make sense are + omitted. The fields are exactly the same as the ones in the <link + linkend="monitoring-pg-stat-io-view"> <structname>pg_stat_io</structname></link> + view. + </para> + </sect2> <sect2 id="monitoring-pg-stat-bgwriter-view"> diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index 7fd5d256a1..b4939b7bc6 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1168,6 +1168,28 @@ SELECT b.stats_reset FROM pg_stat_get_io() b; +CREATE VIEW pg_my_stat_io AS +SELECT + b.backend_type, + b.object, + b.context, + b.reads, + b.read_time, + b.writes, + b.write_time, + b.writebacks, + b.writeback_time, + b.extends, + b.extend_time, + b.op_bytes, + b.hits, + b.evictions, + b.reuses, + b.fsyncs, + b.fsync_time, + b.stats_reset +FROM pg_stat_get_my_io() b; + CREATE VIEW pg_stat_wal AS SELECT w.wal_records, diff --git a/src/backend/utils/activity/pgstat.c b/src/backend/utils/activity/pgstat.c index 178b5ef65a..53b6e0bcd5 100644 --- a/src/backend/utils/activity/pgstat.c +++ b/src/backend/utils/activity/pgstat.c @@ -406,10 +406,12 @@ static const PgStat_KindInfo pgstat_kind_builtin_infos[PGSTAT_KIND_BUILTIN_SIZE] .fixed_amount = true, - .snapshot_ctl_off = offsetof(PgStat_Snapshot, io), + .init_backend_cb = pgstat_io_init_backend_cb, + .snapshot_ctl_off = offsetof(PgStat_Snapshot, global_io), .shared_ctl_off = offsetof(PgStat_ShmemControl, io), - .shared_data_off = offsetof(PgStatShared_IO, stats), - .shared_data_len = sizeof(((PgStatShared_IO *) 0)->stats), + /* [de-]serialize global_stats only */ + .shared_data_off = offsetof(PgStatShared_IO, global_stats), + .shared_data_len = sizeof(((PgStatShared_IO *) 0)->global_stats), .init_shmem_cb = pgstat_io_init_shmem_cb, .reset_all_cb = pgstat_io_reset_all_cb, @@ -581,6 +583,9 @@ pgstat_shutdown_hook(int code, Datum arg) pgstat_report_stat(true); + /* reset the pgstat_io counter related to this proc number */ + pgstat_reset_io_counter_internal(MyProcNumber, GetCurrentTimestamp()); + /* there shouldn't be any pending changes left */ Assert(dlist_is_empty(&pgStatPending)); dlist_init(&pgStatPending); diff --git a/src/backend/utils/activity/pgstat_io.c b/src/backend/utils/activity/pgstat_io.c index 8af55989ee..f2b36d8efa 100644 --- a/src/backend/utils/activity/pgstat_io.c +++ b/src/backend/utils/activity/pgstat_io.c @@ -31,6 +31,7 @@ typedef struct PgStat_PendingIO static PgStat_PendingIO PendingIOStats; bool have_iostats = false; +void pgstat_reset_io_counter_internal(int index, TimestampTz ts); /* * Check that stats have not been counted for any combination of IOObject, @@ -154,11 +155,19 @@ pgstat_count_io_op_time(IOObject io_object, IOContext io_context, IOOp io_op, } PgStat_IO * -pgstat_fetch_stat_io(void) +pgstat_fetch_global_stat_io(void) { pgstat_snapshot_fixed(PGSTAT_KIND_IO); - return &pgStatLocal.snapshot.io; + return &pgStatLocal.snapshot.global_io; +} + +PgStat_Backend_IO * +pgstat_fetch_my_stat_io(void) +{ + pgstat_snapshot_fixed(PGSTAT_KIND_IO); + + return &pgStatLocal.snapshot.my_io; } /* @@ -174,13 +183,16 @@ pgstat_flush_io(bool nowait) { LWLock *bktype_lock; PgStat_BktypeIO *bktype_shstats; + PgStat_BktypeIO *global_bktype_shstats; if (!have_iostats) return false; bktype_lock = &pgStatLocal.shmem->io.locks[MyBackendType]; bktype_shstats = - &pgStatLocal.shmem->io.stats.stats[MyBackendType]; + &pgStatLocal.shmem->io.stat[MyProcNumber].stats; + global_bktype_shstats = + &pgStatLocal.shmem->io.global_stats.stats[MyBackendType]; if (!nowait) LWLockAcquire(bktype_lock, LW_EXCLUSIVE); @@ -194,19 +206,28 @@ pgstat_flush_io(bool nowait) for (int io_op = 0; io_op < IOOP_NUM_TYPES; io_op++) { instr_time time; + PgStat_Counter counter; + + counter = PendingIOStats.counts[io_object][io_context][io_op]; - bktype_shstats->counts[io_object][io_context][io_op] += - PendingIOStats.counts[io_object][io_context][io_op]; + bktype_shstats->counts[io_object][io_context][io_op] += counter; + + global_bktype_shstats->counts[io_object][io_context][io_op] += + counter; time = PendingIOStats.pending_times[io_object][io_context][io_op]; bktype_shstats->times[io_object][io_context][io_op] += INSTR_TIME_GET_MICROSEC(time); + + global_bktype_shstats->times[io_object][io_context][io_op] += + INSTR_TIME_GET_MICROSEC(time); } } } Assert(pgstat_bktype_io_stats_valid(bktype_shstats, MyBackendType)); + Assert(pgstat_bktype_io_stats_valid(global_bktype_shstats, MyBackendType)); LWLockRelease(bktype_lock); @@ -260,13 +281,35 @@ pgstat_io_init_shmem_cb(void *stats) LWLockInitialize(&stat_shmem->locks[i], LWTRANCHE_PGSTATS_DATA); } +void +pgstat_reset_io_counter_internal(int index, TimestampTz ts) +{ + LWLock *bktype_lock = &pgStatLocal.shmem->io.locks[0]; + PgStat_BktypeIO *bktype_shstats = &pgStatLocal.shmem->io.stat[index].stats; + + + /* + * Use the lock in the first BackendType's PgStat_BktypeIO to protect the + * reset timestamp as well. + */ + LWLockAcquire(bktype_lock, LW_EXCLUSIVE); + + pgStatLocal.shmem->io.stat[index].stat_reset_timestamp = ts; + + memset(bktype_shstats, 0, sizeof(*bktype_shstats)); + LWLockRelease(bktype_lock); +} + void pgstat_io_reset_all_cb(TimestampTz ts) { + for (int i = 0; i < NumProcStatSlots; i++) + pgstat_reset_io_counter_internal(i, ts); + for (int i = 0; i < BACKEND_NUM_TYPES; i++) { LWLock *bktype_lock = &pgStatLocal.shmem->io.locks[i]; - PgStat_BktypeIO *bktype_shstats = &pgStatLocal.shmem->io.stats.stats[i]; + PgStat_BktypeIO *bktype_shstats = &pgStatLocal.shmem->io.global_stats.stats[i]; LWLockAcquire(bktype_lock, LW_EXCLUSIVE); @@ -275,7 +318,7 @@ pgstat_io_reset_all_cb(TimestampTz ts) * the reset timestamp as well. */ if (i == 0) - pgStatLocal.shmem->io.stats.stat_reset_timestamp = ts; + pgStatLocal.shmem->io.global_stats.stat_reset_timestamp = ts; memset(bktype_shstats, 0, sizeof(*bktype_shstats)); LWLockRelease(bktype_lock); @@ -285,11 +328,28 @@ pgstat_io_reset_all_cb(TimestampTz ts) void pgstat_io_snapshot_cb(void) { + /* First, our own stats */ + PgStat_BktypeIO *bktype_shstats = &pgStatLocal.shmem->io.stat[MyProcNumber].stats; + PgStat_BktypeIO *bktype_snap = &pgStatLocal.snapshot.my_io.stats; + LWLock *mybktype_lock = &pgStatLocal.shmem->io.locks[0]; + + LWLockAcquire(mybktype_lock, LW_SHARED); + + pgStatLocal.snapshot.my_io.stat_reset_timestamp = + pgStatLocal.shmem->io.stat[MyProcNumber].stat_reset_timestamp; + + pgStatLocal.snapshot.my_io.bktype = + pgStatLocal.shmem->io.stat[MyProcNumber].bktype; + + *bktype_snap = *bktype_shstats; + LWLockRelease(mybktype_lock); + + /* Now, the global stats */ for (int i = 0; i < BACKEND_NUM_TYPES; i++) { LWLock *bktype_lock = &pgStatLocal.shmem->io.locks[i]; - PgStat_BktypeIO *bktype_shstats = &pgStatLocal.shmem->io.stats.stats[i]; - PgStat_BktypeIO *bktype_snap = &pgStatLocal.snapshot.io.stats[i]; + PgStat_BktypeIO *bktype_global_shstats = &pgStatLocal.shmem->io.global_stats.stats[i]; + PgStat_BktypeIO *bktype_global_snap = &pgStatLocal.snapshot.global_io.stats[i]; LWLockAcquire(bktype_lock, LW_SHARED); @@ -298,11 +358,11 @@ pgstat_io_snapshot_cb(void) * the reset timestamp as well. */ if (i == 0) - pgStatLocal.snapshot.io.stat_reset_timestamp = - pgStatLocal.shmem->io.stats.stat_reset_timestamp; + pgStatLocal.snapshot.global_io.stat_reset_timestamp = + pgStatLocal.shmem->io.global_stats.stat_reset_timestamp; /* using struct assignment due to better type safety */ - *bktype_snap = *bktype_shstats; + *bktype_global_snap = *bktype_global_shstats; LWLockRelease(bktype_lock); } } @@ -485,3 +545,10 @@ pgstat_tracks_io_op(BackendType bktype, IOObject io_object, return true; } + +void +pgstat_io_init_backend_cb(void) +{ + /* Initialize our backend type in the IO statistics */ + pgStatLocal.shmem->io.stat[MyProcNumber].bktype = MyBackendType; +} diff --git a/src/backend/utils/activity/pgstat_shmem.c b/src/backend/utils/activity/pgstat_shmem.c index ec93bf6902..3cfcba9609 100644 --- a/src/backend/utils/activity/pgstat_shmem.c +++ b/src/backend/utils/activity/pgstat_shmem.c @@ -128,7 +128,7 @@ StatsShmemSize(void) { Size sz; - sz = MAXALIGN(sizeof(PgStat_ShmemControl)); + sz = MAXALIGN(sizeof(PgStat_ShmemControl) + mul_size(sizeof(PgStat_Backend_IO), NumProcStatSlots)); sz = add_size(sz, pgstat_dsa_init_size()); /* Add shared memory for all the custom fixed-numbered statistics */ @@ -172,7 +172,7 @@ StatsShmemInit(void) Assert(!found); /* the allocation of pgStatLocal.shmem itself */ - p += MAXALIGN(sizeof(PgStat_ShmemControl)); + p += MAXALIGN(sizeof(PgStat_ShmemControl) + mul_size(sizeof(PgStat_Backend_IO), NumProcStatSlots)); /* * Create a small dsa allocation in plain shared memory. This is diff --git a/src/backend/utils/adt/pgstatfuncs.c b/src/backend/utils/adt/pgstatfuncs.c index 97dc09ac0d..e72dc5daa7 100644 --- a/src/backend/utils/adt/pgstatfuncs.c +++ b/src/backend/utils/adt/pgstatfuncs.c @@ -1363,14 +1363,17 @@ pg_stat_get_io(PG_FUNCTION_ARGS) InitMaterializedSRF(fcinfo, 0); rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; - backends_io_stats = pgstat_fetch_stat_io(); + backends_io_stats = pgstat_fetch_global_stat_io(); reset_time = TimestampTzGetDatum(backends_io_stats->stat_reset_timestamp); for (int bktype = 0; bktype < BACKEND_NUM_TYPES; bktype++) { - Datum bktype_desc = CStringGetTextDatum(GetBackendTypeDesc(bktype)); - PgStat_BktypeIO *bktype_stats = &backends_io_stats->stats[bktype]; + Datum bktype_desc; + PgStat_BktypeIO *bktype_stats; + + bktype_desc = CStringGetTextDatum(GetBackendTypeDesc(bktype)); + bktype_stats = &backends_io_stats->stats[bktype]; /* * In Assert builds, we can afford an extra loop through all of the @@ -1462,6 +1465,111 @@ pg_stat_get_io(PG_FUNCTION_ARGS) return (Datum) 0; } +Datum +pg_stat_get_my_io(PG_FUNCTION_ARGS) +{ + ReturnSetInfo *rsinfo; + PgStat_Backend_IO *backend_io_stats; + Datum bktype_desc; + PgStat_BktypeIO *bktype_stats; + BackendType bktype; + Datum reset_time; + + InitMaterializedSRF(fcinfo, 0); + rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; + + backend_io_stats = pgstat_fetch_my_stat_io(); + + bktype = backend_io_stats->bktype; + bktype_desc = CStringGetTextDatum(GetBackendTypeDesc(bktype)); + bktype_stats = &backend_io_stats->stats; + reset_time = TimestampTzGetDatum(backend_io_stats->stat_reset_timestamp); + + /* + * In Assert builds, we can afford an extra loop through all of the + * counters checking that only expected stats are non-zero, since it keeps + * the non-Assert code cleaner. + */ + Assert(pgstat_bktype_io_stats_valid(bktype_stats, bktype)); + + for (int io_obj = 0; io_obj < IOOBJECT_NUM_TYPES; io_obj++) + { + const char *obj_name = pgstat_get_io_object_name(io_obj); + + for (int io_context = 0; io_context < IOCONTEXT_NUM_TYPES; io_context++) + { + const char *context_name = pgstat_get_io_context_name(io_context); + + Datum values[IO_NUM_COLUMNS] = {0}; + bool nulls[IO_NUM_COLUMNS] = {0}; + + /* + * Some combinations of BackendType, IOObject, and IOContext are + * not valid for any type of IOOp. In such cases, omit the entire + * row from the view. + */ + if (!pgstat_tracks_io_object(bktype, io_obj, io_context)) + continue; + + values[IO_COL_BACKEND_TYPE] = bktype_desc; + values[IO_COL_CONTEXT] = CStringGetTextDatum(context_name); + values[IO_COL_OBJECT] = CStringGetTextDatum(obj_name); + if (backend_io_stats->stat_reset_timestamp != 0) + values[IO_COL_RESET_TIME] = reset_time; + else + nulls[IO_COL_RESET_TIME] = true; + + /* + * Hard-code this to the value of BLCKSZ for now. Future values + * could include XLOG_BLCKSZ, once WAL IO is tracked, and constant + * multipliers, once non-block-oriented IO (e.g. temporary file + * IO) is tracked. + */ + values[IO_COL_CONVERSION] = Int64GetDatum(BLCKSZ); + + for (int io_op = 0; io_op < IOOP_NUM_TYPES; io_op++) + { + int op_idx = pgstat_get_io_op_index(io_op); + int time_idx = pgstat_get_io_time_index(io_op); + + /* + * Some combinations of BackendType and IOOp, of IOContext and + * IOOp, and of IOObject and IOOp are not tracked. Set these + * cells in the view NULL. + */ + if (pgstat_tracks_io_op(bktype, io_obj, io_context, io_op)) + { + PgStat_Counter count = + bktype_stats->counts[io_obj][io_context][io_op]; + + values[op_idx] = Int64GetDatum(count); + } + else + nulls[op_idx] = true; + + /* not every operation is timed */ + if (time_idx == IO_COL_INVALID) + continue; + + if (!nulls[op_idx]) + { + PgStat_Counter time = + bktype_stats->times[io_obj][io_context][io_op]; + + values[time_idx] = Float8GetDatum(pg_stat_us_to_ms(time)); + } + else + nulls[time_idx] = true; + } + + tuplestore_putvalues(rsinfo->setResult, rsinfo->setDesc, + values, nulls); + } + } + + return (Datum) 0; +} + /* * Returns statistics of WAL activity */ diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index ff5436acac..62658e9a00 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -5831,6 +5831,15 @@ proargnames => '{backend_type,object,context,reads,read_time,writes,write_time,writebacks,writeback_time,extends,extend_time,op_bytes,hits,evictions,reuses,fsyncs,fsync_time,stats_reset}', prosrc => 'pg_stat_get_io' }, +{ oid => '8806', descr => 'statistics: My backend IO statistics', + proname => 'pg_stat_get_my_io', prorows => '5', proretset => 't', + provolatile => 'v', proparallel => 'r', prorettype => 'record', + proargtypes => '', + proallargtypes => '{text,text,text,int8,float8,int8,float8,int8,float8,int8,float8,int8,int8,int8,int8,int8,float8,timestamptz}', + proargmodes => '{o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o}', + proargnames => '{backend_type,object,context,reads,read_time,writes,write_time,writebacks,writeback_time,extends,extend_time,op_bytes,hits,evictions,reuses,fsyncs,fsync_time,stats_reset}', + prosrc => 'pg_stat_get_my_io' }, + { oid => '1136', descr => 'statistics: information about WAL activity', proname => 'pg_stat_get_wal', proisstrict => 'f', provolatile => 's', proparallel => 'r', prorettype => 'record', proargtypes => '', diff --git a/src/include/pgstat.h b/src/include/pgstat.h index be2c91168a..9d63b1a5b7 100644 --- a/src/include/pgstat.h +++ b/src/include/pgstat.h @@ -16,6 +16,7 @@ #include "portability/instr_time.h" #include "postmaster/pgarch.h" /* for MAX_XFN_CHARS */ #include "replication/conflict.h" +#include "storage/proc.h" #include "utils/backend_progress.h" /* for backward compatibility */ #include "utils/backend_status.h" /* for backward compatibility */ #include "utils/relcache.h" @@ -76,6 +77,8 @@ */ #define PGSTAT_KIND_EXPERIMENTAL 128 +#define NumProcStatSlots (MaxBackends + NUM_AUXILIARY_PROCS) + static inline bool pgstat_is_kind_builtin(PgStat_Kind kind) { @@ -351,6 +354,12 @@ typedef struct PgStat_IO PgStat_BktypeIO stats[BACKEND_NUM_TYPES]; } PgStat_IO; +typedef struct PgStat_Backend_IO +{ + TimestampTz stat_reset_timestamp; + BackendType bktype; + PgStat_BktypeIO stats; +} PgStat_Backend_IO; typedef struct PgStat_StatDBEntry { @@ -559,7 +568,8 @@ extern instr_time pgstat_prepare_io_time(bool track_io_guc); extern void pgstat_count_io_op_time(IOObject io_object, IOContext io_context, IOOp io_op, instr_time start_time, uint32 cnt); -extern PgStat_IO *pgstat_fetch_stat_io(void); +extern PgStat_IO *pgstat_fetch_global_stat_io(void); +extern PgStat_Backend_IO *pgstat_fetch_my_stat_io(void); extern const char *pgstat_get_io_context_name(IOContext io_context); extern const char *pgstat_get_io_object_name(IOObject io_object); @@ -568,7 +578,7 @@ extern bool pgstat_tracks_io_object(BackendType bktype, IOObject io_object, IOContext io_context); extern bool pgstat_tracks_io_op(BackendType bktype, IOObject io_object, IOContext io_context, IOOp io_op); - +extern void pgstat_reset_io_counter_internal(int index, TimestampTz ts); /* * Functions in pgstat_database.c diff --git a/src/include/utils/pgstat_internal.h b/src/include/utils/pgstat_internal.h index 25820cbf0a..3b2f2b28dd 100644 --- a/src/include/utils/pgstat_internal.h +++ b/src/include/utils/pgstat_internal.h @@ -353,11 +353,12 @@ typedef struct PgStatShared_Checkpointer typedef struct PgStatShared_IO { /* - * locks[i] protects stats.stats[i]. locks[0] also protects - * stats.stat_reset_timestamp. + * locks[i] protects global_stats.stats[i]. locks[0] also protects + * global_stats.stat_reset_timestamp. */ LWLock locks[BACKEND_NUM_TYPES]; - PgStat_IO stats; + PgStat_IO global_stats; + PgStat_Backend_IO stat[FLEXIBLE_ARRAY_MEMBER]; } PgStatShared_IO; typedef struct PgStatShared_SLRU @@ -450,7 +451,6 @@ typedef struct PgStat_ShmemControl PgStatShared_Archiver archiver; PgStatShared_BgWriter bgwriter; PgStatShared_Checkpointer checkpointer; - PgStatShared_IO io; PgStatShared_SLRU slru; PgStatShared_Wal wal; @@ -460,6 +460,8 @@ typedef struct PgStat_ShmemControl */ void *custom_data[PGSTAT_KIND_CUSTOM_SIZE]; + /* has to be at the end due to FLEXIBLE_ARRAY_MEMBER */ + PgStatShared_IO io; } PgStat_ShmemControl; @@ -481,7 +483,8 @@ typedef struct PgStat_Snapshot PgStat_CheckpointerStats checkpointer; - PgStat_IO io; + PgStat_Backend_IO my_io; + PgStat_IO global_io; PgStat_SLRUStats slru[SLRU_NUM_ELEMENTS]; @@ -682,6 +685,7 @@ extern bool pgstat_flush_wal(bool nowait); extern bool pgstat_have_pending_wal(void); extern void pgstat_wal_init_backend_cb(void); +extern void pgstat_io_init_backend_cb(void); extern void pgstat_wal_init_shmem_cb(void *stats); extern void pgstat_wal_reset_all_cb(TimestampTz ts); extern void pgstat_wal_snapshot_cb(void); diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out index a1626f3fae..785298d7b0 100644 --- a/src/test/regress/expected/rules.out +++ b/src/test/regress/expected/rules.out @@ -1398,6 +1398,25 @@ pg_matviews| SELECT n.nspname AS schemaname, LEFT JOIN pg_namespace n ON ((n.oid = c.relnamespace))) LEFT JOIN pg_tablespace t ON ((t.oid = c.reltablespace))) WHERE (c.relkind = 'm'::"char"); +pg_my_stat_io| SELECT backend_type, + object, + context, + reads, + read_time, + writes, + write_time, + writebacks, + writeback_time, + extends, + extend_time, + op_bytes, + hits, + evictions, + reuses, + fsyncs, + fsync_time, + stats_reset + FROM pg_stat_get_my_io() b(backend_type, object, context, reads, read_time, writes, write_time, writebacks, writeback_time, extends, extend_time, op_bytes, hits, evictions, reuses, fsyncs, fsync_time, stats_reset); pg_policies| SELECT n.nspname AS schemaname, c.relname AS tablename, pol.polname AS policyname, diff --git a/src/test/regress/expected/stats.out b/src/test/regress/expected/stats.out index 6e08898b18..c489e528e0 100644 --- a/src/test/regress/expected/stats.out +++ b/src/test/regress/expected/stats.out @@ -1249,7 +1249,7 @@ SELECT pg_stat_get_subscription_stats(NULL); (1 row) --- Test that the following operations are tracked in pg_stat_io: +-- Test that the following operations are tracked in pg_[my]_stat_io: -- - reads of target blocks into shared buffers -- - writes of shared buffers to permanent storage -- - extends of relations using shared buffers @@ -1261,9 +1261,14 @@ SELECT pg_stat_get_subscription_stats(NULL); -- extends. SELECT sum(extends) AS io_sum_shared_before_extends FROM pg_stat_io WHERE context = 'normal' AND object = 'relation' \gset +SELECT sum(extends) AS my_io_sum_shared_before_extends + FROM pg_my_stat_io WHERE context = 'normal' AND object = 'relation' \gset SELECT sum(writes) AS writes, sum(fsyncs) AS fsyncs FROM pg_stat_io WHERE object = 'relation' \gset io_sum_shared_before_ +SELECT sum(writes) AS writes, sum(fsyncs) AS fsyncs + FROM pg_my_stat_io + WHERE object = 'relation' \gset my_io_sum_shared_before_ CREATE TABLE test_io_shared(a int); INSERT INTO test_io_shared SELECT i FROM generate_series(1,100)i; SELECT pg_stat_force_next_flush(); @@ -1280,8 +1285,16 @@ SELECT :io_sum_shared_after_extends > :io_sum_shared_before_extends; t (1 row) +SELECT sum(extends) AS my_io_sum_shared_after_extends + FROM pg_my_stat_io WHERE context = 'normal' AND object = 'relation' \gset +SELECT :my_io_sum_shared_after_extends > :my_io_sum_shared_before_extends; + ?column? +---------- + t +(1 row) + -- After a checkpoint, there should be some additional IOCONTEXT_NORMAL writes --- and fsyncs. +-- and fsyncs in the global stats (not for the backend). -- See comment above for rationale for two explicit CHECKPOINTs. CHECKPOINT; CHECKPOINT; @@ -1301,6 +1314,23 @@ SELECT current_setting('fsync') = 'off' t (1 row) +SELECT sum(writes) AS writes, sum(fsyncs) AS fsyncs + FROM pg_my_stat_io + WHERE object = 'relation' \gset my_io_sum_shared_after_ +SELECT :my_io_sum_shared_after_writes >= :my_io_sum_shared_before_writes; + ?column? +---------- + t +(1 row) + +SELECT current_setting('fsync') = 'off' + OR (:my_io_sum_shared_after_fsyncs = :my_io_sum_shared_before_fsyncs + AND :my_io_sum_shared_after_fsyncs= 0); + ?column? +---------- + t +(1 row) + -- Change the tablespace so that the table is rewritten directly, then SELECT -- from it to cause it to be read back into shared buffers. SELECT sum(reads) AS io_sum_shared_before_reads @@ -1521,6 +1551,8 @@ SELECT pg_stat_have_stats('io', 0, 0); SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS io_stats_pre_reset FROM pg_stat_io \gset +SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS my_io_stats_pre_reset + FROM pg_my_stat_io \gset SELECT pg_stat_reset_shared('io'); pg_stat_reset_shared ---------------------- @@ -1535,6 +1567,14 @@ SELECT :io_stats_post_reset < :io_stats_pre_reset; t (1 row) +SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS my_io_stats_post_reset + FROM pg_my_stat_io \gset +SELECT :my_io_stats_post_reset < :my_io_stats_pre_reset; + ?column? +---------- + t +(1 row) + -- test BRIN index doesn't block HOT update CREATE TABLE brin_hot ( id integer PRIMARY KEY, diff --git a/src/test/regress/sql/stats.sql b/src/test/regress/sql/stats.sql index d8ac0d06f4..c95cb71652 100644 --- a/src/test/regress/sql/stats.sql +++ b/src/test/regress/sql/stats.sql @@ -595,7 +595,7 @@ SELECT pg_stat_get_replication_slot(NULL); SELECT pg_stat_get_subscription_stats(NULL); --- Test that the following operations are tracked in pg_stat_io: +-- Test that the following operations are tracked in pg_[my]_stat_io: -- - reads of target blocks into shared buffers -- - writes of shared buffers to permanent storage -- - extends of relations using shared buffers @@ -609,18 +609,26 @@ SELECT pg_stat_get_subscription_stats(NULL); -- extends. SELECT sum(extends) AS io_sum_shared_before_extends FROM pg_stat_io WHERE context = 'normal' AND object = 'relation' \gset +SELECT sum(extends) AS my_io_sum_shared_before_extends + FROM pg_my_stat_io WHERE context = 'normal' AND object = 'relation' \gset SELECT sum(writes) AS writes, sum(fsyncs) AS fsyncs FROM pg_stat_io WHERE object = 'relation' \gset io_sum_shared_before_ +SELECT sum(writes) AS writes, sum(fsyncs) AS fsyncs + FROM pg_my_stat_io + WHERE object = 'relation' \gset my_io_sum_shared_before_ CREATE TABLE test_io_shared(a int); INSERT INTO test_io_shared SELECT i FROM generate_series(1,100)i; SELECT pg_stat_force_next_flush(); SELECT sum(extends) AS io_sum_shared_after_extends FROM pg_stat_io WHERE context = 'normal' AND object = 'relation' \gset SELECT :io_sum_shared_after_extends > :io_sum_shared_before_extends; +SELECT sum(extends) AS my_io_sum_shared_after_extends + FROM pg_my_stat_io WHERE context = 'normal' AND object = 'relation' \gset +SELECT :my_io_sum_shared_after_extends > :my_io_sum_shared_before_extends; -- After a checkpoint, there should be some additional IOCONTEXT_NORMAL writes --- and fsyncs. +-- and fsyncs in the global stats (not for the backend). -- See comment above for rationale for two explicit CHECKPOINTs. CHECKPOINT; CHECKPOINT; @@ -630,7 +638,13 @@ SELECT sum(writes) AS writes, sum(fsyncs) AS fsyncs SELECT :io_sum_shared_after_writes > :io_sum_shared_before_writes; SELECT current_setting('fsync') = 'off' OR :io_sum_shared_after_fsyncs > :io_sum_shared_before_fsyncs; - +SELECT sum(writes) AS writes, sum(fsyncs) AS fsyncs + FROM pg_my_stat_io + WHERE object = 'relation' \gset my_io_sum_shared_after_ +SELECT :my_io_sum_shared_after_writes >= :my_io_sum_shared_before_writes; +SELECT current_setting('fsync') = 'off' + OR (:my_io_sum_shared_after_fsyncs = :my_io_sum_shared_before_fsyncs + AND :my_io_sum_shared_after_fsyncs= 0); -- Change the tablespace so that the table is rewritten directly, then SELECT -- from it to cause it to be read back into shared buffers. SELECT sum(reads) AS io_sum_shared_before_reads @@ -762,10 +776,15 @@ SELECT :io_sum_bulkwrite_strategy_extends_after > :io_sum_bulkwrite_strategy_ext SELECT pg_stat_have_stats('io', 0, 0); SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS io_stats_pre_reset FROM pg_stat_io \gset +SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS my_io_stats_pre_reset + FROM pg_my_stat_io \gset SELECT pg_stat_reset_shared('io'); SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS io_stats_post_reset FROM pg_stat_io \gset SELECT :io_stats_post_reset < :io_stats_pre_reset; +SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS my_io_stats_post_reset + FROM pg_my_stat_io \gset +SELECT :my_io_stats_post_reset < :my_io_stats_pre_reset; -- test BRIN index doesn't block HOT update -- 2.34.1 --0+8wqzuq5PafKmdo Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v2-0002-Add-pg_stat_get_backend_io.patch" ^ permalink raw reply [nested|flat] 25+ messages in thread
* [PATCH v3 1/2] per backend I/O statistics @ 2024-08-22 15:16 Bertrand Drouvot <[email protected]> 0 siblings, 0 replies; 25+ messages in thread From: Bertrand Drouvot @ 2024-08-22 15:16 UTC (permalink / raw) While pg_stat_io provides cluster-wide I/O statistics, this commit adds a new pg_my_stat_io view to display "my" backend I/O statistics. The KIND_IO stats are still "fixed amount" ones as the maximum number of backend is fixed. The statistics snapshot is made for the global stats (the aggregated ones) and for my backend stats. The snapshot is not build in each backend to copy all the others backends stats, as 1/ there is no use case (there is no need to get others backends I/O statistics while taking care of the stats_fetch_consistency) and 2/ that could be memory expensive depending of the number of max connections. A subsequent commit will add a new pg_stat_get_backend_io() function to be able to retrieve the I/O statistics for a given backend pid (each execution re-fetches counters from shared memory because as stated above there is no snapshots being created in each backend to copy the other backends stats). Bump catalog version needs to be done. --- doc/src/sgml/config.sgml | 4 +- doc/src/sgml/monitoring.sgml | 28 ++++++ src/backend/catalog/system_views.sql | 22 +++++ src/backend/utils/activity/pgstat.c | 11 ++- src/backend/utils/activity/pgstat_io.c | 91 ++++++++++++++--- src/backend/utils/activity/pgstat_shmem.c | 4 +- src/backend/utils/adt/pgstatfuncs.c | 114 +++++++++++++++++++++- src/include/catalog/pg_proc.dat | 9 ++ src/include/pgstat.h | 14 ++- src/include/utils/pgstat_internal.h | 14 ++- src/test/regress/expected/rules.out | 19 ++++ src/test/regress/expected/stats.out | 44 ++++++++- src/test/regress/sql/stats.sql | 25 ++++- 13 files changed, 366 insertions(+), 33 deletions(-) 8.5% doc/src/sgml/ 29.4% src/backend/utils/activity/ 23.9% src/backend/utils/adt/ 4.4% src/include/catalog/ 3.1% src/include/utils/ 3.1% src/include/ 14.4% src/test/regress/expected/ 10.0% src/test/regress/sql/ diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 0aec11f443..2a59b97093 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -8331,7 +8331,9 @@ COPY postgres_log FROM '/full/path/to/logfile.csv' WITH csv; displayed in <link linkend="monitoring-pg-stat-database-view"> <structname>pg_stat_database</structname></link>, <link linkend="monitoring-pg-stat-io-view"> - <structname>pg_stat_io</structname></link>, in the output of + <structname>pg_stat_io</structname></link>, + <link linkend="monitoring-pg-my-stat-io-view"> + <structname>pg_my_stat_io</structname></link>, in the output of <xref linkend="sql-explain"/> when the <literal>BUFFERS</literal> option is used, in the output of <xref linkend="sql-vacuum"/> when the <literal>VERBOSE</literal> option is used, by autovacuum diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml index 933de6fe07..b28ca4e0ca 100644 --- a/doc/src/sgml/monitoring.sgml +++ b/doc/src/sgml/monitoring.sgml @@ -488,6 +488,16 @@ postgres 27093 0.0 0.0 30096 2752 ? Ss 11:34 0:00 postgres: ser </entry> </row> + <row> + <entry><structname>pg_my_stat_io</structname><indexterm><primary>pg_my_stat_io</primary></indexterm></entry> + <entry> + One row for each combination of context and target object containing + my backend I/O statistics. + See <link linkend="monitoring-pg-my-stat-io-view"> + <structname>pg_my_stat_io</structname></link> for details. + </entry> + </row> + <row> <entry><structname>pg_stat_replication_slots</structname><indexterm><primary>pg_stat_replication_slots</primary></indexterm></entry> <entry>One row per replication slot, showing statistics about the @@ -2948,6 +2958,24 @@ description | Waiting for a newly initialized WAL file to reach durable storage + </sect2> + + <sect2 id="monitoring-pg-my-stat-io-view"> + <title><structname>pg_my_stat_io</structname></title> + + <indexterm> + <primary>pg_my_stat_io</primary> + </indexterm> + + <para> + The <structname>pg_my_stat_io</structname> view will contain one row for each + combination of target I/O object and I/O context, showing + my backend I/O statistics. Combinations which do not make sense are + omitted. The fields are exactly the same as the ones in the <link + linkend="monitoring-pg-stat-io-view"> <structname>pg_stat_io</structname></link> + view. + </para> + </sect2> <sect2 id="monitoring-pg-stat-bgwriter-view"> diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index 7fd5d256a1..b4939b7bc6 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1168,6 +1168,28 @@ SELECT b.stats_reset FROM pg_stat_get_io() b; +CREATE VIEW pg_my_stat_io AS +SELECT + b.backend_type, + b.object, + b.context, + b.reads, + b.read_time, + b.writes, + b.write_time, + b.writebacks, + b.writeback_time, + b.extends, + b.extend_time, + b.op_bytes, + b.hits, + b.evictions, + b.reuses, + b.fsyncs, + b.fsync_time, + b.stats_reset +FROM pg_stat_get_my_io() b; + CREATE VIEW pg_stat_wal AS SELECT w.wal_records, diff --git a/src/backend/utils/activity/pgstat.c b/src/backend/utils/activity/pgstat.c index a7f2dfc744..deb30cd2fc 100644 --- a/src/backend/utils/activity/pgstat.c +++ b/src/backend/utils/activity/pgstat.c @@ -406,10 +406,12 @@ static const PgStat_KindInfo pgstat_kind_builtin_infos[PGSTAT_KIND_BUILTIN_SIZE] .fixed_amount = true, - .snapshot_ctl_off = offsetof(PgStat_Snapshot, io), + .init_backend_cb = pgstat_io_init_backend_cb, + .snapshot_ctl_off = offsetof(PgStat_Snapshot, global_io), .shared_ctl_off = offsetof(PgStat_ShmemControl, io), - .shared_data_off = offsetof(PgStatShared_IO, stats), - .shared_data_len = sizeof(((PgStatShared_IO *) 0)->stats), + /* [de-]serialize global_stats only */ + .shared_data_off = offsetof(PgStatShared_IO, global_stats), + .shared_data_len = sizeof(((PgStatShared_IO *) 0)->global_stats), .flush_fixed_cb = pgstat_io_flush_cb, .have_fixed_pending_cb = pgstat_io_have_pending_cb, @@ -587,6 +589,9 @@ pgstat_shutdown_hook(int code, Datum arg) pgstat_report_stat(true); + /* reset the pgstat_io counter related to this proc number */ + pgstat_reset_io_counter_internal(MyProcNumber, GetCurrentTimestamp()); + /* there shouldn't be any pending changes left */ Assert(dlist_is_empty(&pgStatPending)); dlist_init(&pgStatPending); diff --git a/src/backend/utils/activity/pgstat_io.c b/src/backend/utils/activity/pgstat_io.c index cc2ffc78aa..611ca21720 100644 --- a/src/backend/utils/activity/pgstat_io.c +++ b/src/backend/utils/activity/pgstat_io.c @@ -31,6 +31,7 @@ typedef struct PgStat_PendingIO static PgStat_PendingIO PendingIOStats; static bool have_iostats = false; +void pgstat_reset_io_counter_internal(int index, TimestampTz ts); /* * Check that stats have not been counted for any combination of IOObject, @@ -154,11 +155,19 @@ pgstat_count_io_op_time(IOObject io_object, IOContext io_context, IOOp io_op, } PgStat_IO * -pgstat_fetch_stat_io(void) +pgstat_fetch_global_stat_io(void) { pgstat_snapshot_fixed(PGSTAT_KIND_IO); - return &pgStatLocal.snapshot.io; + return &pgStatLocal.snapshot.global_io; +} + +PgStat_Backend_IO * +pgstat_fetch_my_stat_io(void) +{ + pgstat_snapshot_fixed(PGSTAT_KIND_IO); + + return &pgStatLocal.snapshot.my_io; } /* @@ -192,13 +201,16 @@ pgstat_io_flush_cb(bool nowait) { LWLock *bktype_lock; PgStat_BktypeIO *bktype_shstats; + PgStat_BktypeIO *global_bktype_shstats; if (!have_iostats) return false; bktype_lock = &pgStatLocal.shmem->io.locks[MyBackendType]; bktype_shstats = - &pgStatLocal.shmem->io.stats.stats[MyBackendType]; + &pgStatLocal.shmem->io.stat[MyProcNumber].stats; + global_bktype_shstats = + &pgStatLocal.shmem->io.global_stats.stats[MyBackendType]; if (!nowait) LWLockAcquire(bktype_lock, LW_EXCLUSIVE); @@ -212,19 +224,28 @@ pgstat_io_flush_cb(bool nowait) for (int io_op = 0; io_op < IOOP_NUM_TYPES; io_op++) { instr_time time; + PgStat_Counter counter; + + counter = PendingIOStats.counts[io_object][io_context][io_op]; - bktype_shstats->counts[io_object][io_context][io_op] += - PendingIOStats.counts[io_object][io_context][io_op]; + bktype_shstats->counts[io_object][io_context][io_op] += counter; + + global_bktype_shstats->counts[io_object][io_context][io_op] += + counter; time = PendingIOStats.pending_times[io_object][io_context][io_op]; bktype_shstats->times[io_object][io_context][io_op] += INSTR_TIME_GET_MICROSEC(time); + + global_bktype_shstats->times[io_object][io_context][io_op] += + INSTR_TIME_GET_MICROSEC(time); } } } Assert(pgstat_bktype_io_stats_valid(bktype_shstats, MyBackendType)); + Assert(pgstat_bktype_io_stats_valid(global_bktype_shstats, MyBackendType)); LWLockRelease(bktype_lock); @@ -278,13 +299,35 @@ pgstat_io_init_shmem_cb(void *stats) LWLockInitialize(&stat_shmem->locks[i], LWTRANCHE_PGSTATS_DATA); } +void +pgstat_reset_io_counter_internal(int index, TimestampTz ts) +{ + LWLock *bktype_lock = &pgStatLocal.shmem->io.locks[0]; + PgStat_BktypeIO *bktype_shstats = &pgStatLocal.shmem->io.stat[index].stats; + + + /* + * Use the lock in the first BackendType's PgStat_BktypeIO to protect the + * reset timestamp as well. + */ + LWLockAcquire(bktype_lock, LW_EXCLUSIVE); + + pgStatLocal.shmem->io.stat[index].stat_reset_timestamp = ts; + + memset(bktype_shstats, 0, sizeof(*bktype_shstats)); + LWLockRelease(bktype_lock); +} + void pgstat_io_reset_all_cb(TimestampTz ts) { + for (int i = 0; i < NumProcStatSlots; i++) + pgstat_reset_io_counter_internal(i, ts); + for (int i = 0; i < BACKEND_NUM_TYPES; i++) { LWLock *bktype_lock = &pgStatLocal.shmem->io.locks[i]; - PgStat_BktypeIO *bktype_shstats = &pgStatLocal.shmem->io.stats.stats[i]; + PgStat_BktypeIO *bktype_shstats = &pgStatLocal.shmem->io.global_stats.stats[i]; LWLockAcquire(bktype_lock, LW_EXCLUSIVE); @@ -293,7 +336,7 @@ pgstat_io_reset_all_cb(TimestampTz ts) * the reset timestamp as well. */ if (i == 0) - pgStatLocal.shmem->io.stats.stat_reset_timestamp = ts; + pgStatLocal.shmem->io.global_stats.stat_reset_timestamp = ts; memset(bktype_shstats, 0, sizeof(*bktype_shstats)); LWLockRelease(bktype_lock); @@ -303,11 +346,28 @@ pgstat_io_reset_all_cb(TimestampTz ts) void pgstat_io_snapshot_cb(void) { + /* First, our own stats */ + PgStat_BktypeIO *bktype_shstats = &pgStatLocal.shmem->io.stat[MyProcNumber].stats; + PgStat_BktypeIO *bktype_snap = &pgStatLocal.snapshot.my_io.stats; + LWLock *mybktype_lock = &pgStatLocal.shmem->io.locks[0]; + + LWLockAcquire(mybktype_lock, LW_SHARED); + + pgStatLocal.snapshot.my_io.stat_reset_timestamp = + pgStatLocal.shmem->io.stat[MyProcNumber].stat_reset_timestamp; + + pgStatLocal.snapshot.my_io.bktype = + pgStatLocal.shmem->io.stat[MyProcNumber].bktype; + + *bktype_snap = *bktype_shstats; + LWLockRelease(mybktype_lock); + + /* Now, the global stats */ for (int i = 0; i < BACKEND_NUM_TYPES; i++) { LWLock *bktype_lock = &pgStatLocal.shmem->io.locks[i]; - PgStat_BktypeIO *bktype_shstats = &pgStatLocal.shmem->io.stats.stats[i]; - PgStat_BktypeIO *bktype_snap = &pgStatLocal.snapshot.io.stats[i]; + PgStat_BktypeIO *bktype_global_shstats = &pgStatLocal.shmem->io.global_stats.stats[i]; + PgStat_BktypeIO *bktype_global_snap = &pgStatLocal.snapshot.global_io.stats[i]; LWLockAcquire(bktype_lock, LW_SHARED); @@ -316,11 +376,11 @@ pgstat_io_snapshot_cb(void) * the reset timestamp as well. */ if (i == 0) - pgStatLocal.snapshot.io.stat_reset_timestamp = - pgStatLocal.shmem->io.stats.stat_reset_timestamp; + pgStatLocal.snapshot.global_io.stat_reset_timestamp = + pgStatLocal.shmem->io.global_stats.stat_reset_timestamp; /* using struct assignment due to better type safety */ - *bktype_snap = *bktype_shstats; + *bktype_global_snap = *bktype_global_shstats; LWLockRelease(bktype_lock); } } @@ -503,3 +563,10 @@ pgstat_tracks_io_op(BackendType bktype, IOObject io_object, return true; } + +void +pgstat_io_init_backend_cb(void) +{ + /* Initialize our backend type in the IO statistics */ + pgStatLocal.shmem->io.stat[MyProcNumber].bktype = MyBackendType; +} diff --git a/src/backend/utils/activity/pgstat_shmem.c b/src/backend/utils/activity/pgstat_shmem.c index ec93bf6902..3cfcba9609 100644 --- a/src/backend/utils/activity/pgstat_shmem.c +++ b/src/backend/utils/activity/pgstat_shmem.c @@ -128,7 +128,7 @@ StatsShmemSize(void) { Size sz; - sz = MAXALIGN(sizeof(PgStat_ShmemControl)); + sz = MAXALIGN(sizeof(PgStat_ShmemControl) + mul_size(sizeof(PgStat_Backend_IO), NumProcStatSlots)); sz = add_size(sz, pgstat_dsa_init_size()); /* Add shared memory for all the custom fixed-numbered statistics */ @@ -172,7 +172,7 @@ StatsShmemInit(void) Assert(!found); /* the allocation of pgStatLocal.shmem itself */ - p += MAXALIGN(sizeof(PgStat_ShmemControl)); + p += MAXALIGN(sizeof(PgStat_ShmemControl) + mul_size(sizeof(PgStat_Backend_IO), NumProcStatSlots)); /* * Create a small dsa allocation in plain shared memory. This is diff --git a/src/backend/utils/adt/pgstatfuncs.c b/src/backend/utils/adt/pgstatfuncs.c index 33c7b25560..4c00570396 100644 --- a/src/backend/utils/adt/pgstatfuncs.c +++ b/src/backend/utils/adt/pgstatfuncs.c @@ -1363,14 +1363,17 @@ pg_stat_get_io(PG_FUNCTION_ARGS) InitMaterializedSRF(fcinfo, 0); rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; - backends_io_stats = pgstat_fetch_stat_io(); + backends_io_stats = pgstat_fetch_global_stat_io(); reset_time = TimestampTzGetDatum(backends_io_stats->stat_reset_timestamp); for (int bktype = 0; bktype < BACKEND_NUM_TYPES; bktype++) { - Datum bktype_desc = CStringGetTextDatum(GetBackendTypeDesc(bktype)); - PgStat_BktypeIO *bktype_stats = &backends_io_stats->stats[bktype]; + Datum bktype_desc; + PgStat_BktypeIO *bktype_stats; + + bktype_desc = CStringGetTextDatum(GetBackendTypeDesc(bktype)); + bktype_stats = &backends_io_stats->stats[bktype]; /* * In Assert builds, we can afford an extra loop through all of the @@ -1462,6 +1465,111 @@ pg_stat_get_io(PG_FUNCTION_ARGS) return (Datum) 0; } +Datum +pg_stat_get_my_io(PG_FUNCTION_ARGS) +{ + ReturnSetInfo *rsinfo; + PgStat_Backend_IO *backend_io_stats; + Datum bktype_desc; + PgStat_BktypeIO *bktype_stats; + BackendType bktype; + Datum reset_time; + + InitMaterializedSRF(fcinfo, 0); + rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; + + backend_io_stats = pgstat_fetch_my_stat_io(); + + bktype = backend_io_stats->bktype; + bktype_desc = CStringGetTextDatum(GetBackendTypeDesc(bktype)); + bktype_stats = &backend_io_stats->stats; + reset_time = TimestampTzGetDatum(backend_io_stats->stat_reset_timestamp); + + /* + * In Assert builds, we can afford an extra loop through all of the + * counters checking that only expected stats are non-zero, since it keeps + * the non-Assert code cleaner. + */ + Assert(pgstat_bktype_io_stats_valid(bktype_stats, bktype)); + + for (int io_obj = 0; io_obj < IOOBJECT_NUM_TYPES; io_obj++) + { + const char *obj_name = pgstat_get_io_object_name(io_obj); + + for (int io_context = 0; io_context < IOCONTEXT_NUM_TYPES; io_context++) + { + const char *context_name = pgstat_get_io_context_name(io_context); + + Datum values[IO_NUM_COLUMNS] = {0}; + bool nulls[IO_NUM_COLUMNS] = {0}; + + /* + * Some combinations of BackendType, IOObject, and IOContext are + * not valid for any type of IOOp. In such cases, omit the entire + * row from the view. + */ + if (!pgstat_tracks_io_object(bktype, io_obj, io_context)) + continue; + + values[IO_COL_BACKEND_TYPE] = bktype_desc; + values[IO_COL_CONTEXT] = CStringGetTextDatum(context_name); + values[IO_COL_OBJECT] = CStringGetTextDatum(obj_name); + if (backend_io_stats->stat_reset_timestamp != 0) + values[IO_COL_RESET_TIME] = reset_time; + else + nulls[IO_COL_RESET_TIME] = true; + + /* + * Hard-code this to the value of BLCKSZ for now. Future values + * could include XLOG_BLCKSZ, once WAL IO is tracked, and constant + * multipliers, once non-block-oriented IO (e.g. temporary file + * IO) is tracked. + */ + values[IO_COL_CONVERSION] = Int64GetDatum(BLCKSZ); + + for (int io_op = 0; io_op < IOOP_NUM_TYPES; io_op++) + { + int op_idx = pgstat_get_io_op_index(io_op); + int time_idx = pgstat_get_io_time_index(io_op); + + /* + * Some combinations of BackendType and IOOp, of IOContext and + * IOOp, and of IOObject and IOOp are not tracked. Set these + * cells in the view NULL. + */ + if (pgstat_tracks_io_op(bktype, io_obj, io_context, io_op)) + { + PgStat_Counter count = + bktype_stats->counts[io_obj][io_context][io_op]; + + values[op_idx] = Int64GetDatum(count); + } + else + nulls[op_idx] = true; + + /* not every operation is timed */ + if (time_idx == IO_COL_INVALID) + continue; + + if (!nulls[op_idx]) + { + PgStat_Counter time = + bktype_stats->times[io_obj][io_context][io_op]; + + values[time_idx] = Float8GetDatum(pg_stat_us_to_ms(time)); + } + else + nulls[time_idx] = true; + } + + tuplestore_putvalues(rsinfo->setResult, rsinfo->setDesc, + values, nulls); + } + } + + return (Datum) 0; +} + /* * Returns statistics of WAL activity */ diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index ff5436acac..62658e9a00 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -5831,6 +5831,15 @@ proargnames => '{backend_type,object,context,reads,read_time,writes,write_time,writebacks,writeback_time,extends,extend_time,op_bytes,hits,evictions,reuses,fsyncs,fsync_time,stats_reset}', prosrc => 'pg_stat_get_io' }, +{ oid => '8806', descr => 'statistics: My backend IO statistics', + proname => 'pg_stat_get_my_io', prorows => '5', proretset => 't', + provolatile => 'v', proparallel => 'r', prorettype => 'record', + proargtypes => '', + proallargtypes => '{text,text,text,int8,float8,int8,float8,int8,float8,int8,float8,int8,int8,int8,int8,int8,float8,timestamptz}', + proargmodes => '{o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o}', + proargnames => '{backend_type,object,context,reads,read_time,writes,write_time,writebacks,writeback_time,extends,extend_time,op_bytes,hits,evictions,reuses,fsyncs,fsync_time,stats_reset}', + prosrc => 'pg_stat_get_my_io' }, + { oid => '1136', descr => 'statistics: information about WAL activity', proname => 'pg_stat_get_wal', proisstrict => 'f', provolatile => 's', proparallel => 'r', prorettype => 'record', proargtypes => '', diff --git a/src/include/pgstat.h b/src/include/pgstat.h index be2c91168a..9d63b1a5b7 100644 --- a/src/include/pgstat.h +++ b/src/include/pgstat.h @@ -16,6 +16,7 @@ #include "portability/instr_time.h" #include "postmaster/pgarch.h" /* for MAX_XFN_CHARS */ #include "replication/conflict.h" +#include "storage/proc.h" #include "utils/backend_progress.h" /* for backward compatibility */ #include "utils/backend_status.h" /* for backward compatibility */ #include "utils/relcache.h" @@ -76,6 +77,8 @@ */ #define PGSTAT_KIND_EXPERIMENTAL 128 +#define NumProcStatSlots (MaxBackends + NUM_AUXILIARY_PROCS) + static inline bool pgstat_is_kind_builtin(PgStat_Kind kind) { @@ -351,6 +354,12 @@ typedef struct PgStat_IO PgStat_BktypeIO stats[BACKEND_NUM_TYPES]; } PgStat_IO; +typedef struct PgStat_Backend_IO +{ + TimestampTz stat_reset_timestamp; + BackendType bktype; + PgStat_BktypeIO stats; +} PgStat_Backend_IO; typedef struct PgStat_StatDBEntry { @@ -559,7 +568,8 @@ extern instr_time pgstat_prepare_io_time(bool track_io_guc); extern void pgstat_count_io_op_time(IOObject io_object, IOContext io_context, IOOp io_op, instr_time start_time, uint32 cnt); -extern PgStat_IO *pgstat_fetch_stat_io(void); +extern PgStat_IO *pgstat_fetch_global_stat_io(void); +extern PgStat_Backend_IO *pgstat_fetch_my_stat_io(void); extern const char *pgstat_get_io_context_name(IOContext io_context); extern const char *pgstat_get_io_object_name(IOObject io_object); @@ -568,7 +578,7 @@ extern bool pgstat_tracks_io_object(BackendType bktype, IOObject io_object, IOContext io_context); extern bool pgstat_tracks_io_op(BackendType bktype, IOObject io_object, IOContext io_context, IOOp io_op); - +extern void pgstat_reset_io_counter_internal(int index, TimestampTz ts); /* * Functions in pgstat_database.c diff --git a/src/include/utils/pgstat_internal.h b/src/include/utils/pgstat_internal.h index bba90e898d..ea4cc6a2c6 100644 --- a/src/include/utils/pgstat_internal.h +++ b/src/include/utils/pgstat_internal.h @@ -366,11 +366,12 @@ typedef struct PgStatShared_Checkpointer typedef struct PgStatShared_IO { /* - * locks[i] protects stats.stats[i]. locks[0] also protects - * stats.stat_reset_timestamp. + * locks[i] protects global_stats.stats[i]. locks[0] also protects + * global_stats.stat_reset_timestamp. */ LWLock locks[BACKEND_NUM_TYPES]; - PgStat_IO stats; + PgStat_IO global_stats; + PgStat_Backend_IO stat[FLEXIBLE_ARRAY_MEMBER]; } PgStatShared_IO; typedef struct PgStatShared_SLRU @@ -463,7 +464,6 @@ typedef struct PgStat_ShmemControl PgStatShared_Archiver archiver; PgStatShared_BgWriter bgwriter; PgStatShared_Checkpointer checkpointer; - PgStatShared_IO io; PgStatShared_SLRU slru; PgStatShared_Wal wal; @@ -473,6 +473,8 @@ typedef struct PgStat_ShmemControl */ void *custom_data[PGSTAT_KIND_CUSTOM_SIZE]; + /* has to be at the end due to FLEXIBLE_ARRAY_MEMBER */ + PgStatShared_IO io; } PgStat_ShmemControl; @@ -494,7 +496,8 @@ typedef struct PgStat_Snapshot PgStat_CheckpointerStats checkpointer; - PgStat_IO io; + PgStat_Backend_IO my_io; + PgStat_IO global_io; PgStat_SLRUStats slru[SLRU_NUM_ELEMENTS]; @@ -629,6 +632,7 @@ extern bool pgstat_io_flush_cb(bool nowait); extern void pgstat_io_init_shmem_cb(void *stats); extern void pgstat_io_reset_all_cb(TimestampTz ts); extern void pgstat_io_snapshot_cb(void); +extern void pgstat_io_init_backend_cb(void); /* diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out index a1626f3fae..785298d7b0 100644 --- a/src/test/regress/expected/rules.out +++ b/src/test/regress/expected/rules.out @@ -1398,6 +1398,25 @@ pg_matviews| SELECT n.nspname AS schemaname, LEFT JOIN pg_namespace n ON ((n.oid = c.relnamespace))) LEFT JOIN pg_tablespace t ON ((t.oid = c.reltablespace))) WHERE (c.relkind = 'm'::"char"); +pg_my_stat_io| SELECT backend_type, + object, + context, + reads, + read_time, + writes, + write_time, + writebacks, + writeback_time, + extends, + extend_time, + op_bytes, + hits, + evictions, + reuses, + fsyncs, + fsync_time, + stats_reset + FROM pg_stat_get_my_io() b(backend_type, object, context, reads, read_time, writes, write_time, writebacks, writeback_time, extends, extend_time, op_bytes, hits, evictions, reuses, fsyncs, fsync_time, stats_reset); pg_policies| SELECT n.nspname AS schemaname, c.relname AS tablename, pol.polname AS policyname, diff --git a/src/test/regress/expected/stats.out b/src/test/regress/expected/stats.out index 6e08898b18..c489e528e0 100644 --- a/src/test/regress/expected/stats.out +++ b/src/test/regress/expected/stats.out @@ -1249,7 +1249,7 @@ SELECT pg_stat_get_subscription_stats(NULL); (1 row) --- Test that the following operations are tracked in pg_stat_io: +-- Test that the following operations are tracked in pg_[my]_stat_io: -- - reads of target blocks into shared buffers -- - writes of shared buffers to permanent storage -- - extends of relations using shared buffers @@ -1261,9 +1261,14 @@ SELECT pg_stat_get_subscription_stats(NULL); -- extends. SELECT sum(extends) AS io_sum_shared_before_extends FROM pg_stat_io WHERE context = 'normal' AND object = 'relation' \gset +SELECT sum(extends) AS my_io_sum_shared_before_extends + FROM pg_my_stat_io WHERE context = 'normal' AND object = 'relation' \gset SELECT sum(writes) AS writes, sum(fsyncs) AS fsyncs FROM pg_stat_io WHERE object = 'relation' \gset io_sum_shared_before_ +SELECT sum(writes) AS writes, sum(fsyncs) AS fsyncs + FROM pg_my_stat_io + WHERE object = 'relation' \gset my_io_sum_shared_before_ CREATE TABLE test_io_shared(a int); INSERT INTO test_io_shared SELECT i FROM generate_series(1,100)i; SELECT pg_stat_force_next_flush(); @@ -1280,8 +1285,16 @@ SELECT :io_sum_shared_after_extends > :io_sum_shared_before_extends; t (1 row) +SELECT sum(extends) AS my_io_sum_shared_after_extends + FROM pg_my_stat_io WHERE context = 'normal' AND object = 'relation' \gset +SELECT :my_io_sum_shared_after_extends > :my_io_sum_shared_before_extends; + ?column? +---------- + t +(1 row) + -- After a checkpoint, there should be some additional IOCONTEXT_NORMAL writes --- and fsyncs. +-- and fsyncs in the global stats (not for the backend). -- See comment above for rationale for two explicit CHECKPOINTs. CHECKPOINT; CHECKPOINT; @@ -1301,6 +1314,23 @@ SELECT current_setting('fsync') = 'off' t (1 row) +SELECT sum(writes) AS writes, sum(fsyncs) AS fsyncs + FROM pg_my_stat_io + WHERE object = 'relation' \gset my_io_sum_shared_after_ +SELECT :my_io_sum_shared_after_writes >= :my_io_sum_shared_before_writes; + ?column? +---------- + t +(1 row) + +SELECT current_setting('fsync') = 'off' + OR (:my_io_sum_shared_after_fsyncs = :my_io_sum_shared_before_fsyncs + AND :my_io_sum_shared_after_fsyncs= 0); + ?column? +---------- + t +(1 row) + -- Change the tablespace so that the table is rewritten directly, then SELECT -- from it to cause it to be read back into shared buffers. SELECT sum(reads) AS io_sum_shared_before_reads @@ -1521,6 +1551,8 @@ SELECT pg_stat_have_stats('io', 0, 0); SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS io_stats_pre_reset FROM pg_stat_io \gset +SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS my_io_stats_pre_reset + FROM pg_my_stat_io \gset SELECT pg_stat_reset_shared('io'); pg_stat_reset_shared ---------------------- @@ -1535,6 +1567,14 @@ SELECT :io_stats_post_reset < :io_stats_pre_reset; t (1 row) +SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS my_io_stats_post_reset + FROM pg_my_stat_io \gset +SELECT :my_io_stats_post_reset < :my_io_stats_pre_reset; + ?column? +---------- + t +(1 row) + -- test BRIN index doesn't block HOT update CREATE TABLE brin_hot ( id integer PRIMARY KEY, diff --git a/src/test/regress/sql/stats.sql b/src/test/regress/sql/stats.sql index d8ac0d06f4..c95cb71652 100644 --- a/src/test/regress/sql/stats.sql +++ b/src/test/regress/sql/stats.sql @@ -595,7 +595,7 @@ SELECT pg_stat_get_replication_slot(NULL); SELECT pg_stat_get_subscription_stats(NULL); --- Test that the following operations are tracked in pg_stat_io: +-- Test that the following operations are tracked in pg_[my]_stat_io: -- - reads of target blocks into shared buffers -- - writes of shared buffers to permanent storage -- - extends of relations using shared buffers @@ -609,18 +609,26 @@ SELECT pg_stat_get_subscription_stats(NULL); -- extends. SELECT sum(extends) AS io_sum_shared_before_extends FROM pg_stat_io WHERE context = 'normal' AND object = 'relation' \gset +SELECT sum(extends) AS my_io_sum_shared_before_extends + FROM pg_my_stat_io WHERE context = 'normal' AND object = 'relation' \gset SELECT sum(writes) AS writes, sum(fsyncs) AS fsyncs FROM pg_stat_io WHERE object = 'relation' \gset io_sum_shared_before_ +SELECT sum(writes) AS writes, sum(fsyncs) AS fsyncs + FROM pg_my_stat_io + WHERE object = 'relation' \gset my_io_sum_shared_before_ CREATE TABLE test_io_shared(a int); INSERT INTO test_io_shared SELECT i FROM generate_series(1,100)i; SELECT pg_stat_force_next_flush(); SELECT sum(extends) AS io_sum_shared_after_extends FROM pg_stat_io WHERE context = 'normal' AND object = 'relation' \gset SELECT :io_sum_shared_after_extends > :io_sum_shared_before_extends; +SELECT sum(extends) AS my_io_sum_shared_after_extends + FROM pg_my_stat_io WHERE context = 'normal' AND object = 'relation' \gset +SELECT :my_io_sum_shared_after_extends > :my_io_sum_shared_before_extends; -- After a checkpoint, there should be some additional IOCONTEXT_NORMAL writes --- and fsyncs. +-- and fsyncs in the global stats (not for the backend). -- See comment above for rationale for two explicit CHECKPOINTs. CHECKPOINT; CHECKPOINT; @@ -630,7 +638,13 @@ SELECT sum(writes) AS writes, sum(fsyncs) AS fsyncs SELECT :io_sum_shared_after_writes > :io_sum_shared_before_writes; SELECT current_setting('fsync') = 'off' OR :io_sum_shared_after_fsyncs > :io_sum_shared_before_fsyncs; - +SELECT sum(writes) AS writes, sum(fsyncs) AS fsyncs + FROM pg_my_stat_io + WHERE object = 'relation' \gset my_io_sum_shared_after_ +SELECT :my_io_sum_shared_after_writes >= :my_io_sum_shared_before_writes; +SELECT current_setting('fsync') = 'off' + OR (:my_io_sum_shared_after_fsyncs = :my_io_sum_shared_before_fsyncs + AND :my_io_sum_shared_after_fsyncs= 0); -- Change the tablespace so that the table is rewritten directly, then SELECT -- from it to cause it to be read back into shared buffers. SELECT sum(reads) AS io_sum_shared_before_reads @@ -762,10 +776,15 @@ SELECT :io_sum_bulkwrite_strategy_extends_after > :io_sum_bulkwrite_strategy_ext SELECT pg_stat_have_stats('io', 0, 0); SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS io_stats_pre_reset FROM pg_stat_io \gset +SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS my_io_stats_pre_reset + FROM pg_my_stat_io \gset SELECT pg_stat_reset_shared('io'); SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS io_stats_post_reset FROM pg_stat_io \gset SELECT :io_stats_post_reset < :io_stats_pre_reset; +SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS my_io_stats_post_reset + FROM pg_my_stat_io \gset +SELECT :my_io_stats_post_reset < :my_io_stats_pre_reset; -- test BRIN index doesn't block HOT update -- 2.34.1 --zQrxP4xQumq+YTTN Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v3-0002-Add-pg_stat_get_backend_io.patch" ^ permalink raw reply [nested|flat] 25+ messages in thread
* [PATCH v4 1/2] per backend I/O statistics @ 2024-08-22 15:16 Bertrand Drouvot <[email protected]> 0 siblings, 0 replies; 25+ messages in thread From: Bertrand Drouvot @ 2024-08-22 15:16 UTC (permalink / raw) While pg_stat_io provides cluster-wide I/O statistics, this commit adds a new pg_my_stat_io view to display "my" backend I/O statistics. The KIND_IO stats are still "fixed amount" ones as the maximum number of backend is fixed. The statistics snapshot is made for the global stats (the aggregated ones) and for my backend stats. The snapshot is not build in each backend to copy all the others backends stats, as 1/ there is no use case (there is no need to get others backends I/O statistics while taking care of the stats_fetch_consistency) and 2/ that could be memory expensive depending of the number of max connections. A subsequent commit will add a new pg_stat_get_backend_io() function to be able to retrieve the I/O statistics for a given backend pid (each execution re-fetches counters from shared memory because as stated above there is no snapshots being created in each backend to copy the other backends stats). XXX: Bump catalog version needs to be done. --- doc/src/sgml/config.sgml | 4 +- doc/src/sgml/monitoring.sgml | 28 ++++++ src/backend/catalog/system_views.sql | 22 +++++ src/backend/utils/activity/pgstat.c | 11 ++- src/backend/utils/activity/pgstat_io.c | 93 +++++++++++++++--- src/backend/utils/activity/pgstat_shmem.c | 4 +- src/backend/utils/adt/pgstatfuncs.c | 114 +++++++++++++++++++++- src/include/catalog/pg_proc.dat | 9 ++ src/include/pgstat.h | 14 ++- src/include/utils/pgstat_internal.h | 14 ++- src/test/regress/expected/rules.out | 19 ++++ src/test/regress/expected/stats.out | 44 ++++++++- src/test/regress/sql/stats.sql | 25 ++++- 13 files changed, 368 insertions(+), 33 deletions(-) 8.4% doc/src/sgml/ 29.9% src/backend/utils/activity/ 23.7% src/backend/utils/adt/ 4.4% src/include/catalog/ 3.1% src/include/utils/ 3.1% src/include/ 14.3% src/test/regress/expected/ 9.9% src/test/regress/sql/ diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 0aec11f443..2a59b97093 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -8331,7 +8331,9 @@ COPY postgres_log FROM '/full/path/to/logfile.csv' WITH csv; displayed in <link linkend="monitoring-pg-stat-database-view"> <structname>pg_stat_database</structname></link>, <link linkend="monitoring-pg-stat-io-view"> - <structname>pg_stat_io</structname></link>, in the output of + <structname>pg_stat_io</structname></link>, + <link linkend="monitoring-pg-my-stat-io-view"> + <structname>pg_my_stat_io</structname></link>, in the output of <xref linkend="sql-explain"/> when the <literal>BUFFERS</literal> option is used, in the output of <xref linkend="sql-vacuum"/> when the <literal>VERBOSE</literal> option is used, by autovacuum diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml index 933de6fe07..b28ca4e0ca 100644 --- a/doc/src/sgml/monitoring.sgml +++ b/doc/src/sgml/monitoring.sgml @@ -488,6 +488,16 @@ postgres 27093 0.0 0.0 30096 2752 ? Ss 11:34 0:00 postgres: ser </entry> </row> + <row> + <entry><structname>pg_my_stat_io</structname><indexterm><primary>pg_my_stat_io</primary></indexterm></entry> + <entry> + One row for each combination of context and target object containing + my backend I/O statistics. + See <link linkend="monitoring-pg-my-stat-io-view"> + <structname>pg_my_stat_io</structname></link> for details. + </entry> + </row> + <row> <entry><structname>pg_stat_replication_slots</structname><indexterm><primary>pg_stat_replication_slots</primary></indexterm></entry> <entry>One row per replication slot, showing statistics about the @@ -2948,6 +2958,24 @@ description | Waiting for a newly initialized WAL file to reach durable storage + </sect2> + + <sect2 id="monitoring-pg-my-stat-io-view"> + <title><structname>pg_my_stat_io</structname></title> + + <indexterm> + <primary>pg_my_stat_io</primary> + </indexterm> + + <para> + The <structname>pg_my_stat_io</structname> view will contain one row for each + combination of target I/O object and I/O context, showing + my backend I/O statistics. Combinations which do not make sense are + omitted. The fields are exactly the same as the ones in the <link + linkend="monitoring-pg-stat-io-view"> <structname>pg_stat_io</structname></link> + view. + </para> + </sect2> <sect2 id="monitoring-pg-stat-bgwriter-view"> diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index 7fd5d256a1..b4939b7bc6 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1168,6 +1168,28 @@ SELECT b.stats_reset FROM pg_stat_get_io() b; +CREATE VIEW pg_my_stat_io AS +SELECT + b.backend_type, + b.object, + b.context, + b.reads, + b.read_time, + b.writes, + b.write_time, + b.writebacks, + b.writeback_time, + b.extends, + b.extend_time, + b.op_bytes, + b.hits, + b.evictions, + b.reuses, + b.fsyncs, + b.fsync_time, + b.stats_reset +FROM pg_stat_get_my_io() b; + CREATE VIEW pg_stat_wal AS SELECT w.wal_records, diff --git a/src/backend/utils/activity/pgstat.c b/src/backend/utils/activity/pgstat.c index a7f2dfc744..fda4ca9a4a 100644 --- a/src/backend/utils/activity/pgstat.c +++ b/src/backend/utils/activity/pgstat.c @@ -406,10 +406,12 @@ static const PgStat_KindInfo pgstat_kind_builtin_infos[PGSTAT_KIND_BUILTIN_SIZE] .fixed_amount = true, - .snapshot_ctl_off = offsetof(PgStat_Snapshot, io), + .init_backend_cb = pgstat_io_init_backend_cb, + .snapshot_ctl_off = offsetof(PgStat_Snapshot, global_io), .shared_ctl_off = offsetof(PgStat_ShmemControl, io), - .shared_data_off = offsetof(PgStatShared_IO, stats), - .shared_data_len = sizeof(((PgStatShared_IO *) 0)->stats), + /* [de-]serialize global_stats only */ + .shared_data_off = offsetof(PgStatShared_IO, global_stats), + .shared_data_len = sizeof(((PgStatShared_IO *) 0)->global_stats), .flush_fixed_cb = pgstat_io_flush_cb, .have_fixed_pending_cb = pgstat_io_have_pending_cb, @@ -587,6 +589,9 @@ pgstat_shutdown_hook(int code, Datum arg) pgstat_report_stat(true); + /* reset the pgstat_io counter related to this proc number */ + pgstat_reset_io_counter_internal(MyProcNumber, 0); + /* there shouldn't be any pending changes left */ Assert(dlist_is_empty(&pgStatPending)); dlist_init(&pgStatPending); diff --git a/src/backend/utils/activity/pgstat_io.c b/src/backend/utils/activity/pgstat_io.c index cc2ffc78aa..d32121fc1f 100644 --- a/src/backend/utils/activity/pgstat_io.c +++ b/src/backend/utils/activity/pgstat_io.c @@ -31,6 +31,7 @@ typedef struct PgStat_PendingIO static PgStat_PendingIO PendingIOStats; static bool have_iostats = false; +void pgstat_reset_io_counter_internal(int index, TimestampTz ts); /* * Check that stats have not been counted for any combination of IOObject, @@ -154,11 +155,19 @@ pgstat_count_io_op_time(IOObject io_object, IOContext io_context, IOOp io_op, } PgStat_IO * -pgstat_fetch_stat_io(void) +pgstat_fetch_global_stat_io(void) { pgstat_snapshot_fixed(PGSTAT_KIND_IO); - return &pgStatLocal.snapshot.io; + return &pgStatLocal.snapshot.global_io; +} + +PgStat_Backend_IO * +pgstat_fetch_my_stat_io(void) +{ + pgstat_snapshot_fixed(PGSTAT_KIND_IO); + + return &pgStatLocal.snapshot.my_io; } /* @@ -192,13 +201,16 @@ pgstat_io_flush_cb(bool nowait) { LWLock *bktype_lock; PgStat_BktypeIO *bktype_shstats; + PgStat_BktypeIO *global_bktype_shstats; if (!have_iostats) return false; bktype_lock = &pgStatLocal.shmem->io.locks[MyBackendType]; bktype_shstats = - &pgStatLocal.shmem->io.stats.stats[MyBackendType]; + &pgStatLocal.shmem->io.stat[MyProcNumber].stats; + global_bktype_shstats = + &pgStatLocal.shmem->io.global_stats.stats[MyBackendType]; if (!nowait) LWLockAcquire(bktype_lock, LW_EXCLUSIVE); @@ -212,19 +224,28 @@ pgstat_io_flush_cb(bool nowait) for (int io_op = 0; io_op < IOOP_NUM_TYPES; io_op++) { instr_time time; + PgStat_Counter counter; + + counter = PendingIOStats.counts[io_object][io_context][io_op]; - bktype_shstats->counts[io_object][io_context][io_op] += - PendingIOStats.counts[io_object][io_context][io_op]; + bktype_shstats->counts[io_object][io_context][io_op] += counter; + + global_bktype_shstats->counts[io_object][io_context][io_op] += + counter; time = PendingIOStats.pending_times[io_object][io_context][io_op]; bktype_shstats->times[io_object][io_context][io_op] += INSTR_TIME_GET_MICROSEC(time); + + global_bktype_shstats->times[io_object][io_context][io_op] += + INSTR_TIME_GET_MICROSEC(time); } } } Assert(pgstat_bktype_io_stats_valid(bktype_shstats, MyBackendType)); + Assert(pgstat_bktype_io_stats_valid(global_bktype_shstats, MyBackendType)); LWLockRelease(bktype_lock); @@ -278,13 +299,35 @@ pgstat_io_init_shmem_cb(void *stats) LWLockInitialize(&stat_shmem->locks[i], LWTRANCHE_PGSTATS_DATA); } +void +pgstat_reset_io_counter_internal(int index, TimestampTz ts) +{ + LWLock *bktype_lock = &pgStatLocal.shmem->io.locks[0]; + PgStat_BktypeIO *bktype_shstats = &pgStatLocal.shmem->io.stat[index].stats; + + + /* + * Use the lock in the first BackendType's PgStat_BktypeIO to protect the + * reset timestamp as well. + */ + LWLockAcquire(bktype_lock, LW_EXCLUSIVE); + + pgStatLocal.shmem->io.stat[index].stat_reset_timestamp = ts; + + memset(bktype_shstats, 0, sizeof(*bktype_shstats)); + LWLockRelease(bktype_lock); +} + void pgstat_io_reset_all_cb(TimestampTz ts) { + for (int i = 0; i < NumProcStatSlots; i++) + pgstat_reset_io_counter_internal(i, ts); + for (int i = 0; i < BACKEND_NUM_TYPES; i++) { LWLock *bktype_lock = &pgStatLocal.shmem->io.locks[i]; - PgStat_BktypeIO *bktype_shstats = &pgStatLocal.shmem->io.stats.stats[i]; + PgStat_BktypeIO *bktype_shstats = &pgStatLocal.shmem->io.global_stats.stats[i]; LWLockAcquire(bktype_lock, LW_EXCLUSIVE); @@ -293,7 +336,7 @@ pgstat_io_reset_all_cb(TimestampTz ts) * the reset timestamp as well. */ if (i == 0) - pgStatLocal.shmem->io.stats.stat_reset_timestamp = ts; + pgStatLocal.shmem->io.global_stats.stat_reset_timestamp = ts; memset(bktype_shstats, 0, sizeof(*bktype_shstats)); LWLockRelease(bktype_lock); @@ -303,11 +346,28 @@ pgstat_io_reset_all_cb(TimestampTz ts) void pgstat_io_snapshot_cb(void) { + /* First, our own stats */ + PgStat_BktypeIO *bktype_shstats = &pgStatLocal.shmem->io.stat[MyProcNumber].stats; + PgStat_BktypeIO *bktype_snap = &pgStatLocal.snapshot.my_io.stats; + LWLock *mybktype_lock = &pgStatLocal.shmem->io.locks[0]; + + LWLockAcquire(mybktype_lock, LW_SHARED); + + pgStatLocal.snapshot.my_io.stat_reset_timestamp = + pgStatLocal.shmem->io.stat[MyProcNumber].stat_reset_timestamp; + + pgStatLocal.snapshot.my_io.bktype = + pgStatLocal.shmem->io.stat[MyProcNumber].bktype; + + *bktype_snap = *bktype_shstats; + LWLockRelease(mybktype_lock); + + /* Now, the global stats */ for (int i = 0; i < BACKEND_NUM_TYPES; i++) { LWLock *bktype_lock = &pgStatLocal.shmem->io.locks[i]; - PgStat_BktypeIO *bktype_shstats = &pgStatLocal.shmem->io.stats.stats[i]; - PgStat_BktypeIO *bktype_snap = &pgStatLocal.snapshot.io.stats[i]; + PgStat_BktypeIO *bktype_global_shstats = &pgStatLocal.shmem->io.global_stats.stats[i]; + PgStat_BktypeIO *bktype_global_snap = &pgStatLocal.snapshot.global_io.stats[i]; LWLockAcquire(bktype_lock, LW_SHARED); @@ -316,11 +376,11 @@ pgstat_io_snapshot_cb(void) * the reset timestamp as well. */ if (i == 0) - pgStatLocal.snapshot.io.stat_reset_timestamp = - pgStatLocal.shmem->io.stats.stat_reset_timestamp; + pgStatLocal.snapshot.global_io.stat_reset_timestamp = + pgStatLocal.shmem->io.global_stats.stat_reset_timestamp; /* using struct assignment due to better type safety */ - *bktype_snap = *bktype_shstats; + *bktype_global_snap = *bktype_global_shstats; LWLockRelease(bktype_lock); } } @@ -503,3 +563,12 @@ pgstat_tracks_io_op(BackendType bktype, IOObject io_object, return true; } + +void +pgstat_io_init_backend_cb(void) +{ + /* Initialize our backend type in the IO statistics */ + pgStatLocal.shmem->io.stat[MyProcNumber].bktype = MyBackendType; + /* Set the stat_reset_timestamp to zero */ + pgStatLocal.shmem->io.stat[MyProcNumber].stat_reset_timestamp = 0; +} diff --git a/src/backend/utils/activity/pgstat_shmem.c b/src/backend/utils/activity/pgstat_shmem.c index ec93bf6902..3cfcba9609 100644 --- a/src/backend/utils/activity/pgstat_shmem.c +++ b/src/backend/utils/activity/pgstat_shmem.c @@ -128,7 +128,7 @@ StatsShmemSize(void) { Size sz; - sz = MAXALIGN(sizeof(PgStat_ShmemControl)); + sz = MAXALIGN(sizeof(PgStat_ShmemControl) + mul_size(sizeof(PgStat_Backend_IO), NumProcStatSlots)); sz = add_size(sz, pgstat_dsa_init_size()); /* Add shared memory for all the custom fixed-numbered statistics */ @@ -172,7 +172,7 @@ StatsShmemInit(void) Assert(!found); /* the allocation of pgStatLocal.shmem itself */ - p += MAXALIGN(sizeof(PgStat_ShmemControl)); + p += MAXALIGN(sizeof(PgStat_ShmemControl) + mul_size(sizeof(PgStat_Backend_IO), NumProcStatSlots)); /* * Create a small dsa allocation in plain shared memory. This is diff --git a/src/backend/utils/adt/pgstatfuncs.c b/src/backend/utils/adt/pgstatfuncs.c index 33c7b25560..4c00570396 100644 --- a/src/backend/utils/adt/pgstatfuncs.c +++ b/src/backend/utils/adt/pgstatfuncs.c @@ -1363,14 +1363,17 @@ pg_stat_get_io(PG_FUNCTION_ARGS) InitMaterializedSRF(fcinfo, 0); rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; - backends_io_stats = pgstat_fetch_stat_io(); + backends_io_stats = pgstat_fetch_global_stat_io(); reset_time = TimestampTzGetDatum(backends_io_stats->stat_reset_timestamp); for (int bktype = 0; bktype < BACKEND_NUM_TYPES; bktype++) { - Datum bktype_desc = CStringGetTextDatum(GetBackendTypeDesc(bktype)); - PgStat_BktypeIO *bktype_stats = &backends_io_stats->stats[bktype]; + Datum bktype_desc; + PgStat_BktypeIO *bktype_stats; + + bktype_desc = CStringGetTextDatum(GetBackendTypeDesc(bktype)); + bktype_stats = &backends_io_stats->stats[bktype]; /* * In Assert builds, we can afford an extra loop through all of the @@ -1462,6 +1465,111 @@ pg_stat_get_io(PG_FUNCTION_ARGS) return (Datum) 0; } +Datum +pg_stat_get_my_io(PG_FUNCTION_ARGS) +{ + ReturnSetInfo *rsinfo; + PgStat_Backend_IO *backend_io_stats; + Datum bktype_desc; + PgStat_BktypeIO *bktype_stats; + BackendType bktype; + Datum reset_time; + + InitMaterializedSRF(fcinfo, 0); + rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; + + backend_io_stats = pgstat_fetch_my_stat_io(); + + bktype = backend_io_stats->bktype; + bktype_desc = CStringGetTextDatum(GetBackendTypeDesc(bktype)); + bktype_stats = &backend_io_stats->stats; + reset_time = TimestampTzGetDatum(backend_io_stats->stat_reset_timestamp); + + /* + * In Assert builds, we can afford an extra loop through all of the + * counters checking that only expected stats are non-zero, since it keeps + * the non-Assert code cleaner. + */ + Assert(pgstat_bktype_io_stats_valid(bktype_stats, bktype)); + + for (int io_obj = 0; io_obj < IOOBJECT_NUM_TYPES; io_obj++) + { + const char *obj_name = pgstat_get_io_object_name(io_obj); + + for (int io_context = 0; io_context < IOCONTEXT_NUM_TYPES; io_context++) + { + const char *context_name = pgstat_get_io_context_name(io_context); + + Datum values[IO_NUM_COLUMNS] = {0}; + bool nulls[IO_NUM_COLUMNS] = {0}; + + /* + * Some combinations of BackendType, IOObject, and IOContext are + * not valid for any type of IOOp. In such cases, omit the entire + * row from the view. + */ + if (!pgstat_tracks_io_object(bktype, io_obj, io_context)) + continue; + + values[IO_COL_BACKEND_TYPE] = bktype_desc; + values[IO_COL_CONTEXT] = CStringGetTextDatum(context_name); + values[IO_COL_OBJECT] = CStringGetTextDatum(obj_name); + if (backend_io_stats->stat_reset_timestamp != 0) + values[IO_COL_RESET_TIME] = reset_time; + else + nulls[IO_COL_RESET_TIME] = true; + + /* + * Hard-code this to the value of BLCKSZ for now. Future values + * could include XLOG_BLCKSZ, once WAL IO is tracked, and constant + * multipliers, once non-block-oriented IO (e.g. temporary file + * IO) is tracked. + */ + values[IO_COL_CONVERSION] = Int64GetDatum(BLCKSZ); + + for (int io_op = 0; io_op < IOOP_NUM_TYPES; io_op++) + { + int op_idx = pgstat_get_io_op_index(io_op); + int time_idx = pgstat_get_io_time_index(io_op); + + /* + * Some combinations of BackendType and IOOp, of IOContext and + * IOOp, and of IOObject and IOOp are not tracked. Set these + * cells in the view NULL. + */ + if (pgstat_tracks_io_op(bktype, io_obj, io_context, io_op)) + { + PgStat_Counter count = + bktype_stats->counts[io_obj][io_context][io_op]; + + values[op_idx] = Int64GetDatum(count); + } + else + nulls[op_idx] = true; + + /* not every operation is timed */ + if (time_idx == IO_COL_INVALID) + continue; + + if (!nulls[op_idx]) + { + PgStat_Counter time = + bktype_stats->times[io_obj][io_context][io_op]; + + values[time_idx] = Float8GetDatum(pg_stat_us_to_ms(time)); + } + else + nulls[time_idx] = true; + } + + tuplestore_putvalues(rsinfo->setResult, rsinfo->setDesc, + values, nulls); + } + } + + return (Datum) 0; +} + /* * Returns statistics of WAL activity */ diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index 53a081ed88..4d5af3d7cf 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -5880,6 +5880,15 @@ proargnames => '{backend_type,object,context,reads,read_time,writes,write_time,writebacks,writeback_time,extends,extend_time,op_bytes,hits,evictions,reuses,fsyncs,fsync_time,stats_reset}', prosrc => 'pg_stat_get_io' }, +{ oid => '8806', descr => 'statistics: My backend IO statistics', + proname => 'pg_stat_get_my_io', prorows => '5', proretset => 't', + provolatile => 'v', proparallel => 'r', prorettype => 'record', + proargtypes => '', + proallargtypes => '{text,text,text,int8,float8,int8,float8,int8,float8,int8,float8,int8,int8,int8,int8,int8,float8,timestamptz}', + proargmodes => '{o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o}', + proargnames => '{backend_type,object,context,reads,read_time,writes,write_time,writebacks,writeback_time,extends,extend_time,op_bytes,hits,evictions,reuses,fsyncs,fsync_time,stats_reset}', + prosrc => 'pg_stat_get_my_io' }, + { oid => '1136', descr => 'statistics: information about WAL activity', proname => 'pg_stat_get_wal', proisstrict => 'f', provolatile => 's', proparallel => 'r', prorettype => 'record', proargtypes => '', diff --git a/src/include/pgstat.h b/src/include/pgstat.h index be2c91168a..9d63b1a5b7 100644 --- a/src/include/pgstat.h +++ b/src/include/pgstat.h @@ -16,6 +16,7 @@ #include "portability/instr_time.h" #include "postmaster/pgarch.h" /* for MAX_XFN_CHARS */ #include "replication/conflict.h" +#include "storage/proc.h" #include "utils/backend_progress.h" /* for backward compatibility */ #include "utils/backend_status.h" /* for backward compatibility */ #include "utils/relcache.h" @@ -76,6 +77,8 @@ */ #define PGSTAT_KIND_EXPERIMENTAL 128 +#define NumProcStatSlots (MaxBackends + NUM_AUXILIARY_PROCS) + static inline bool pgstat_is_kind_builtin(PgStat_Kind kind) { @@ -351,6 +354,12 @@ typedef struct PgStat_IO PgStat_BktypeIO stats[BACKEND_NUM_TYPES]; } PgStat_IO; +typedef struct PgStat_Backend_IO +{ + TimestampTz stat_reset_timestamp; + BackendType bktype; + PgStat_BktypeIO stats; +} PgStat_Backend_IO; typedef struct PgStat_StatDBEntry { @@ -559,7 +568,8 @@ extern instr_time pgstat_prepare_io_time(bool track_io_guc); extern void pgstat_count_io_op_time(IOObject io_object, IOContext io_context, IOOp io_op, instr_time start_time, uint32 cnt); -extern PgStat_IO *pgstat_fetch_stat_io(void); +extern PgStat_IO *pgstat_fetch_global_stat_io(void); +extern PgStat_Backend_IO *pgstat_fetch_my_stat_io(void); extern const char *pgstat_get_io_context_name(IOContext io_context); extern const char *pgstat_get_io_object_name(IOObject io_object); @@ -568,7 +578,7 @@ extern bool pgstat_tracks_io_object(BackendType bktype, IOObject io_object, IOContext io_context); extern bool pgstat_tracks_io_op(BackendType bktype, IOObject io_object, IOContext io_context, IOOp io_op); - +extern void pgstat_reset_io_counter_internal(int index, TimestampTz ts); /* * Functions in pgstat_database.c diff --git a/src/include/utils/pgstat_internal.h b/src/include/utils/pgstat_internal.h index bba90e898d..ea4cc6a2c6 100644 --- a/src/include/utils/pgstat_internal.h +++ b/src/include/utils/pgstat_internal.h @@ -366,11 +366,12 @@ typedef struct PgStatShared_Checkpointer typedef struct PgStatShared_IO { /* - * locks[i] protects stats.stats[i]. locks[0] also protects - * stats.stat_reset_timestamp. + * locks[i] protects global_stats.stats[i]. locks[0] also protects + * global_stats.stat_reset_timestamp. */ LWLock locks[BACKEND_NUM_TYPES]; - PgStat_IO stats; + PgStat_IO global_stats; + PgStat_Backend_IO stat[FLEXIBLE_ARRAY_MEMBER]; } PgStatShared_IO; typedef struct PgStatShared_SLRU @@ -463,7 +464,6 @@ typedef struct PgStat_ShmemControl PgStatShared_Archiver archiver; PgStatShared_BgWriter bgwriter; PgStatShared_Checkpointer checkpointer; - PgStatShared_IO io; PgStatShared_SLRU slru; PgStatShared_Wal wal; @@ -473,6 +473,8 @@ typedef struct PgStat_ShmemControl */ void *custom_data[PGSTAT_KIND_CUSTOM_SIZE]; + /* has to be at the end due to FLEXIBLE_ARRAY_MEMBER */ + PgStatShared_IO io; } PgStat_ShmemControl; @@ -494,7 +496,8 @@ typedef struct PgStat_Snapshot PgStat_CheckpointerStats checkpointer; - PgStat_IO io; + PgStat_Backend_IO my_io; + PgStat_IO global_io; PgStat_SLRUStats slru[SLRU_NUM_ELEMENTS]; @@ -629,6 +632,7 @@ extern bool pgstat_io_flush_cb(bool nowait); extern void pgstat_io_init_shmem_cb(void *stats); extern void pgstat_io_reset_all_cb(TimestampTz ts); extern void pgstat_io_snapshot_cb(void); +extern void pgstat_io_init_backend_cb(void); /* diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out index a1626f3fae..785298d7b0 100644 --- a/src/test/regress/expected/rules.out +++ b/src/test/regress/expected/rules.out @@ -1398,6 +1398,25 @@ pg_matviews| SELECT n.nspname AS schemaname, LEFT JOIN pg_namespace n ON ((n.oid = c.relnamespace))) LEFT JOIN pg_tablespace t ON ((t.oid = c.reltablespace))) WHERE (c.relkind = 'm'::"char"); +pg_my_stat_io| SELECT backend_type, + object, + context, + reads, + read_time, + writes, + write_time, + writebacks, + writeback_time, + extends, + extend_time, + op_bytes, + hits, + evictions, + reuses, + fsyncs, + fsync_time, + stats_reset + FROM pg_stat_get_my_io() b(backend_type, object, context, reads, read_time, writes, write_time, writebacks, writeback_time, extends, extend_time, op_bytes, hits, evictions, reuses, fsyncs, fsync_time, stats_reset); pg_policies| SELECT n.nspname AS schemaname, c.relname AS tablename, pol.polname AS policyname, diff --git a/src/test/regress/expected/stats.out b/src/test/regress/expected/stats.out index 6e08898b18..c489e528e0 100644 --- a/src/test/regress/expected/stats.out +++ b/src/test/regress/expected/stats.out @@ -1249,7 +1249,7 @@ SELECT pg_stat_get_subscription_stats(NULL); (1 row) --- Test that the following operations are tracked in pg_stat_io: +-- Test that the following operations are tracked in pg_[my]_stat_io: -- - reads of target blocks into shared buffers -- - writes of shared buffers to permanent storage -- - extends of relations using shared buffers @@ -1261,9 +1261,14 @@ SELECT pg_stat_get_subscription_stats(NULL); -- extends. SELECT sum(extends) AS io_sum_shared_before_extends FROM pg_stat_io WHERE context = 'normal' AND object = 'relation' \gset +SELECT sum(extends) AS my_io_sum_shared_before_extends + FROM pg_my_stat_io WHERE context = 'normal' AND object = 'relation' \gset SELECT sum(writes) AS writes, sum(fsyncs) AS fsyncs FROM pg_stat_io WHERE object = 'relation' \gset io_sum_shared_before_ +SELECT sum(writes) AS writes, sum(fsyncs) AS fsyncs + FROM pg_my_stat_io + WHERE object = 'relation' \gset my_io_sum_shared_before_ CREATE TABLE test_io_shared(a int); INSERT INTO test_io_shared SELECT i FROM generate_series(1,100)i; SELECT pg_stat_force_next_flush(); @@ -1280,8 +1285,16 @@ SELECT :io_sum_shared_after_extends > :io_sum_shared_before_extends; t (1 row) +SELECT sum(extends) AS my_io_sum_shared_after_extends + FROM pg_my_stat_io WHERE context = 'normal' AND object = 'relation' \gset +SELECT :my_io_sum_shared_after_extends > :my_io_sum_shared_before_extends; + ?column? +---------- + t +(1 row) + -- After a checkpoint, there should be some additional IOCONTEXT_NORMAL writes --- and fsyncs. +-- and fsyncs in the global stats (not for the backend). -- See comment above for rationale for two explicit CHECKPOINTs. CHECKPOINT; CHECKPOINT; @@ -1301,6 +1314,23 @@ SELECT current_setting('fsync') = 'off' t (1 row) +SELECT sum(writes) AS writes, sum(fsyncs) AS fsyncs + FROM pg_my_stat_io + WHERE object = 'relation' \gset my_io_sum_shared_after_ +SELECT :my_io_sum_shared_after_writes >= :my_io_sum_shared_before_writes; + ?column? +---------- + t +(1 row) + +SELECT current_setting('fsync') = 'off' + OR (:my_io_sum_shared_after_fsyncs = :my_io_sum_shared_before_fsyncs + AND :my_io_sum_shared_after_fsyncs= 0); + ?column? +---------- + t +(1 row) + -- Change the tablespace so that the table is rewritten directly, then SELECT -- from it to cause it to be read back into shared buffers. SELECT sum(reads) AS io_sum_shared_before_reads @@ -1521,6 +1551,8 @@ SELECT pg_stat_have_stats('io', 0, 0); SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS io_stats_pre_reset FROM pg_stat_io \gset +SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS my_io_stats_pre_reset + FROM pg_my_stat_io \gset SELECT pg_stat_reset_shared('io'); pg_stat_reset_shared ---------------------- @@ -1535,6 +1567,14 @@ SELECT :io_stats_post_reset < :io_stats_pre_reset; t (1 row) +SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS my_io_stats_post_reset + FROM pg_my_stat_io \gset +SELECT :my_io_stats_post_reset < :my_io_stats_pre_reset; + ?column? +---------- + t +(1 row) + -- test BRIN index doesn't block HOT update CREATE TABLE brin_hot ( id integer PRIMARY KEY, diff --git a/src/test/regress/sql/stats.sql b/src/test/regress/sql/stats.sql index d8ac0d06f4..c95cb71652 100644 --- a/src/test/regress/sql/stats.sql +++ b/src/test/regress/sql/stats.sql @@ -595,7 +595,7 @@ SELECT pg_stat_get_replication_slot(NULL); SELECT pg_stat_get_subscription_stats(NULL); --- Test that the following operations are tracked in pg_stat_io: +-- Test that the following operations are tracked in pg_[my]_stat_io: -- - reads of target blocks into shared buffers -- - writes of shared buffers to permanent storage -- - extends of relations using shared buffers @@ -609,18 +609,26 @@ SELECT pg_stat_get_subscription_stats(NULL); -- extends. SELECT sum(extends) AS io_sum_shared_before_extends FROM pg_stat_io WHERE context = 'normal' AND object = 'relation' \gset +SELECT sum(extends) AS my_io_sum_shared_before_extends + FROM pg_my_stat_io WHERE context = 'normal' AND object = 'relation' \gset SELECT sum(writes) AS writes, sum(fsyncs) AS fsyncs FROM pg_stat_io WHERE object = 'relation' \gset io_sum_shared_before_ +SELECT sum(writes) AS writes, sum(fsyncs) AS fsyncs + FROM pg_my_stat_io + WHERE object = 'relation' \gset my_io_sum_shared_before_ CREATE TABLE test_io_shared(a int); INSERT INTO test_io_shared SELECT i FROM generate_series(1,100)i; SELECT pg_stat_force_next_flush(); SELECT sum(extends) AS io_sum_shared_after_extends FROM pg_stat_io WHERE context = 'normal' AND object = 'relation' \gset SELECT :io_sum_shared_after_extends > :io_sum_shared_before_extends; +SELECT sum(extends) AS my_io_sum_shared_after_extends + FROM pg_my_stat_io WHERE context = 'normal' AND object = 'relation' \gset +SELECT :my_io_sum_shared_after_extends > :my_io_sum_shared_before_extends; -- After a checkpoint, there should be some additional IOCONTEXT_NORMAL writes --- and fsyncs. +-- and fsyncs in the global stats (not for the backend). -- See comment above for rationale for two explicit CHECKPOINTs. CHECKPOINT; CHECKPOINT; @@ -630,7 +638,13 @@ SELECT sum(writes) AS writes, sum(fsyncs) AS fsyncs SELECT :io_sum_shared_after_writes > :io_sum_shared_before_writes; SELECT current_setting('fsync') = 'off' OR :io_sum_shared_after_fsyncs > :io_sum_shared_before_fsyncs; - +SELECT sum(writes) AS writes, sum(fsyncs) AS fsyncs + FROM pg_my_stat_io + WHERE object = 'relation' \gset my_io_sum_shared_after_ +SELECT :my_io_sum_shared_after_writes >= :my_io_sum_shared_before_writes; +SELECT current_setting('fsync') = 'off' + OR (:my_io_sum_shared_after_fsyncs = :my_io_sum_shared_before_fsyncs + AND :my_io_sum_shared_after_fsyncs= 0); -- Change the tablespace so that the table is rewritten directly, then SELECT -- from it to cause it to be read back into shared buffers. SELECT sum(reads) AS io_sum_shared_before_reads @@ -762,10 +776,15 @@ SELECT :io_sum_bulkwrite_strategy_extends_after > :io_sum_bulkwrite_strategy_ext SELECT pg_stat_have_stats('io', 0, 0); SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS io_stats_pre_reset FROM pg_stat_io \gset +SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS my_io_stats_pre_reset + FROM pg_my_stat_io \gset SELECT pg_stat_reset_shared('io'); SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS io_stats_post_reset FROM pg_stat_io \gset SELECT :io_stats_post_reset < :io_stats_pre_reset; +SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS my_io_stats_post_reset + FROM pg_my_stat_io \gset +SELECT :my_io_stats_post_reset < :my_io_stats_pre_reset; -- test BRIN index doesn't block HOT update -- 2.34.1 --pJYTS7leEEf4d5MD Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v4-0002-Add-pg_stat_get_backend_io.patch" ^ permalink raw reply [nested|flat] 25+ messages in thread
* [PATCH v1 1/2] per backend I/O statistics @ 2024-08-22 15:16 Bertrand Drouvot <[email protected]> 0 siblings, 0 replies; 25+ messages in thread From: Bertrand Drouvot @ 2024-08-22 15:16 UTC (permalink / raw) While pg_stat_io provides cluster-wide I/O statistics, this commit adds a new pg_my_stat_io view to display "my" backend I/O statistics. The KIND_IO stats are still "fixed amount" ones as the maximum number of backend is fixed. The statistics snapshot is made for the global stats (the aggregated ones) and for my backend stats. The snapshot is not build for all the backend stats (that could be memory expensive depending of the number of max connections and given the fact that PgStat_IO is 16K bytes long). A subsequent commit will add a new pg_stat_get_backend_io() function to be able to retrieve the I/O statistics for a given backend pid. Bump catalog version. --- doc/src/sgml/config.sgml | 4 +- doc/src/sgml/monitoring.sgml | 28 +++++++++ src/backend/catalog/system_views.sql | 24 +++++++- src/backend/utils/activity/pgstat.c | 10 ++- src/backend/utils/activity/pgstat_io.c | 75 +++++++++++++++++++---- src/backend/utils/activity/pgstat_shmem.c | 4 +- src/backend/utils/adt/pgstatfuncs.c | 18 +++++- src/include/catalog/catversion.h | 2 +- src/include/catalog/pg_proc.dat | 8 +-- src/include/pgstat.h | 8 ++- src/include/utils/pgstat_internal.h | 13 ++-- src/test/regress/expected/rules.out | 21 ++++++- src/test/regress/expected/stats.out | 44 ++++++++++++- src/test/regress/sql/stats.sql | 25 +++++++- 14 files changed, 245 insertions(+), 39 deletions(-) 10.7% doc/src/sgml/ 4.1% src/backend/catalog/ 30.9% src/backend/utils/activity/ 5.2% src/backend/utils/adt/ 7.9% src/include/catalog/ 3.4% src/include/utils/ 22.0% src/test/regress/expected/ 12.7% src/test/regress/sql/ diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 0aec11f443..2a59b97093 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -8331,7 +8331,9 @@ COPY postgres_log FROM '/full/path/to/logfile.csv' WITH csv; displayed in <link linkend="monitoring-pg-stat-database-view"> <structname>pg_stat_database</structname></link>, <link linkend="monitoring-pg-stat-io-view"> - <structname>pg_stat_io</structname></link>, in the output of + <structname>pg_stat_io</structname></link>, + <link linkend="monitoring-pg-my-stat-io-view"> + <structname>pg_my_stat_io</structname></link>, in the output of <xref linkend="sql-explain"/> when the <literal>BUFFERS</literal> option is used, in the output of <xref linkend="sql-vacuum"/> when the <literal>VERBOSE</literal> option is used, by autovacuum diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml index 55417a6fa9..27d2548d61 100644 --- a/doc/src/sgml/monitoring.sgml +++ b/doc/src/sgml/monitoring.sgml @@ -488,6 +488,16 @@ postgres 27093 0.0 0.0 30096 2752 ? Ss 11:34 0:00 postgres: ser </entry> </row> + <row> + <entry><structname>pg_my_stat_io</structname><indexterm><primary>pg_my_stat_io</primary></indexterm></entry> + <entry> + One row for each combination of context and target object containing + my backend I/O statistics. + See <link linkend="monitoring-pg-my-stat-io-view"> + <structname>pg_my_stat_io</structname></link> for details. + </entry> + </row> + <row> <entry><structname>pg_stat_replication_slots</structname><indexterm><primary>pg_stat_replication_slots</primary></indexterm></entry> <entry>One row per replication slot, showing statistics about the @@ -2875,6 +2885,24 @@ description | Waiting for a newly initialized WAL file to reach durable storage + </sect2> + + <sect2 id="monitoring-pg-my-stat-io-view"> + <title><structname>pg_my_stat_io</structname></title> + + <indexterm> + <primary>pg_my_stat_io</primary> + </indexterm> + + <para> + The <structname>pg_my_stat_io</structname> view will contain one row for each + combination of target I/O object and I/O context, showing + my backend I/O statistics. Combinations which do not make sense are + omitted. The fields are exactly the same as the ones in the <link + linkend="monitoring-pg-stat-io-view"> <structname>pg_stat_io</structname></link> + view. + </para> + </sect2> <sect2 id="monitoring-pg-stat-bgwriter-view"> diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index 19cabc9a47..7fa050893a 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1166,7 +1166,29 @@ SELECT b.fsyncs, b.fsync_time, b.stats_reset -FROM pg_stat_get_io() b; +FROM pg_stat_get_io(true) b; + +CREATE VIEW pg_my_stat_io AS +SELECT + b.backend_type, + b.object, + b.context, + b.reads, + b.read_time, + b.writes, + b.write_time, + b.writebacks, + b.writeback_time, + b.extends, + b.extend_time, + b.op_bytes, + b.hits, + b.evictions, + b.reuses, + b.fsyncs, + b.fsync_time, + b.stats_reset +FROM pg_stat_get_io(false) b; CREATE VIEW pg_stat_wal AS SELECT diff --git a/src/backend/utils/activity/pgstat.c b/src/backend/utils/activity/pgstat.c index b2ca3f39b7..b0905e6bf1 100644 --- a/src/backend/utils/activity/pgstat.c +++ b/src/backend/utils/activity/pgstat.c @@ -406,10 +406,11 @@ static const PgStat_KindInfo pgstat_kind_builtin_infos[PGSTAT_KIND_BUILTIN_SIZE] .fixed_amount = true, - .snapshot_ctl_off = offsetof(PgStat_Snapshot, io), + .snapshot_ctl_off = offsetof(PgStat_Snapshot, global_io), .shared_ctl_off = offsetof(PgStat_ShmemControl, io), - .shared_data_off = offsetof(PgStatShared_IO, stats), - .shared_data_len = sizeof(((PgStatShared_IO *) 0)->stats), + /* [de-]serialize global_stats only */ + .shared_data_off = offsetof(PgStatShared_IO, global_stats), + .shared_data_len = sizeof(((PgStatShared_IO *) 0)->global_stats), .init_shmem_cb = pgstat_io_init_shmem_cb, .reset_all_cb = pgstat_io_reset_all_cb, @@ -580,6 +581,9 @@ pgstat_shutdown_hook(int code, Datum arg) pgstat_report_stat(true); + /* reset the pgstat_io counter related to this proc number */ + pgstat_reset_io_counter_internal(MyProcNumber, GetCurrentTimestamp()); + /* there shouldn't be any pending changes left */ Assert(dlist_is_empty(&pgStatPending)); dlist_init(&pgStatPending); diff --git a/src/backend/utils/activity/pgstat_io.c b/src/backend/utils/activity/pgstat_io.c index 8af55989ee..43280f4892 100644 --- a/src/backend/utils/activity/pgstat_io.c +++ b/src/backend/utils/activity/pgstat_io.c @@ -31,6 +31,7 @@ typedef struct PgStat_PendingIO static PgStat_PendingIO PendingIOStats; bool have_iostats = false; +void pgstat_reset_io_counter_internal(int index, TimestampTz ts); /* * Check that stats have not been counted for any combination of IOObject, @@ -154,11 +155,19 @@ pgstat_count_io_op_time(IOObject io_object, IOContext io_context, IOOp io_op, } PgStat_IO * -pgstat_fetch_stat_io(void) +pgstat_fetch_global_stat_io(void) { pgstat_snapshot_fixed(PGSTAT_KIND_IO); - return &pgStatLocal.snapshot.io; + return &pgStatLocal.snapshot.global_io; +} + +PgStat_IO * +pgstat_fetch_my_stat_io(void) +{ + pgstat_snapshot_fixed(PGSTAT_KIND_IO); + + return &pgStatLocal.snapshot.my_io; } /* @@ -174,13 +183,16 @@ pgstat_flush_io(bool nowait) { LWLock *bktype_lock; PgStat_BktypeIO *bktype_shstats; + PgStat_BktypeIO *global_bktype_shstats; if (!have_iostats) return false; bktype_lock = &pgStatLocal.shmem->io.locks[MyBackendType]; bktype_shstats = - &pgStatLocal.shmem->io.stats.stats[MyBackendType]; + &pgStatLocal.shmem->io.stats[MyProcNumber].stats[MyBackendType]; + global_bktype_shstats = + &pgStatLocal.shmem->io.global_stats.stats[MyBackendType]; if (!nowait) LWLockAcquire(bktype_lock, LW_EXCLUSIVE); @@ -194,19 +206,28 @@ pgstat_flush_io(bool nowait) for (int io_op = 0; io_op < IOOP_NUM_TYPES; io_op++) { instr_time time; + PgStat_Counter counter; + + counter = PendingIOStats.counts[io_object][io_context][io_op]; + + bktype_shstats->counts[io_object][io_context][io_op] += counter; - bktype_shstats->counts[io_object][io_context][io_op] += - PendingIOStats.counts[io_object][io_context][io_op]; + global_bktype_shstats->counts[io_object][io_context][io_op] += + counter; time = PendingIOStats.pending_times[io_object][io_context][io_op]; bktype_shstats->times[io_object][io_context][io_op] += INSTR_TIME_GET_MICROSEC(time); + + global_bktype_shstats->times[io_object][io_context][io_op] += + INSTR_TIME_GET_MICROSEC(time); } } } Assert(pgstat_bktype_io_stats_valid(bktype_shstats, MyBackendType)); + Assert(pgstat_bktype_io_stats_valid(global_bktype_shstats, MyBackendType)); LWLockRelease(bktype_lock); @@ -260,13 +281,38 @@ pgstat_io_init_shmem_cb(void *stats) LWLockInitialize(&stat_shmem->locks[i], LWTRANCHE_PGSTATS_DATA); } +void +pgstat_reset_io_counter_internal(int index, TimestampTz ts) +{ + for (int i = 0; i < BACKEND_NUM_TYPES; i++) + { + LWLock *bktype_lock = &pgStatLocal.shmem->io.locks[i]; + PgStat_BktypeIO *bktype_shstats = &pgStatLocal.shmem->io.stats[index].stats[i]; + + LWLockAcquire(bktype_lock, LW_EXCLUSIVE); + + /* + * Use the lock in the first BackendType's PgStat_BktypeIO to protect + * the reset timestamp as well. + */ + if (i == 0) + pgStatLocal.shmem->io.stats[index].stat_reset_timestamp = ts; + + memset(bktype_shstats, 0, sizeof(*bktype_shstats)); + LWLockRelease(bktype_lock); + } +} + void pgstat_io_reset_all_cb(TimestampTz ts) { + for (int i = 0; i < NumProcStatSlots; i++) + pgstat_reset_io_counter_internal(i, ts); + for (int i = 0; i < BACKEND_NUM_TYPES; i++) { LWLock *bktype_lock = &pgStatLocal.shmem->io.locks[i]; - PgStat_BktypeIO *bktype_shstats = &pgStatLocal.shmem->io.stats.stats[i]; + PgStat_BktypeIO *bktype_shstats = &pgStatLocal.shmem->io.global_stats.stats[i]; LWLockAcquire(bktype_lock, LW_EXCLUSIVE); @@ -275,7 +321,7 @@ pgstat_io_reset_all_cb(TimestampTz ts) * the reset timestamp as well. */ if (i == 0) - pgStatLocal.shmem->io.stats.stat_reset_timestamp = ts; + pgStatLocal.shmem->io.global_stats.stat_reset_timestamp = ts; memset(bktype_shstats, 0, sizeof(*bktype_shstats)); LWLockRelease(bktype_lock); @@ -288,8 +334,10 @@ pgstat_io_snapshot_cb(void) for (int i = 0; i < BACKEND_NUM_TYPES; i++) { LWLock *bktype_lock = &pgStatLocal.shmem->io.locks[i]; - PgStat_BktypeIO *bktype_shstats = &pgStatLocal.shmem->io.stats.stats[i]; - PgStat_BktypeIO *bktype_snap = &pgStatLocal.snapshot.io.stats[i]; + PgStat_BktypeIO *bktype_global_shstats = &pgStatLocal.shmem->io.global_stats.stats[i]; + PgStat_BktypeIO *bktype_global_snap = &pgStatLocal.snapshot.global_io.stats[i]; + PgStat_BktypeIO *bktype_shstats = &pgStatLocal.shmem->io.stats[MyProcNumber].stats[i]; + PgStat_BktypeIO *bktype_snap = &pgStatLocal.snapshot.my_io.stats[i]; LWLockAcquire(bktype_lock, LW_SHARED); @@ -298,10 +346,15 @@ pgstat_io_snapshot_cb(void) * the reset timestamp as well. */ if (i == 0) - pgStatLocal.snapshot.io.stat_reset_timestamp = - pgStatLocal.shmem->io.stats.stat_reset_timestamp; + { + pgStatLocal.snapshot.global_io.stat_reset_timestamp = + pgStatLocal.shmem->io.global_stats.stat_reset_timestamp; + pgStatLocal.snapshot.my_io.stat_reset_timestamp = + pgStatLocal.shmem->io.stats[MyProcNumber].stat_reset_timestamp; + } /* using struct assignment due to better type safety */ + *bktype_global_snap = *bktype_global_shstats; *bktype_snap = *bktype_shstats; LWLockRelease(bktype_lock); } diff --git a/src/backend/utils/activity/pgstat_shmem.c b/src/backend/utils/activity/pgstat_shmem.c index ec93bf6902..7cf1c64f9e 100644 --- a/src/backend/utils/activity/pgstat_shmem.c +++ b/src/backend/utils/activity/pgstat_shmem.c @@ -128,7 +128,7 @@ StatsShmemSize(void) { Size sz; - sz = MAXALIGN(sizeof(PgStat_ShmemControl)); + sz = MAXALIGN(sizeof(PgStat_ShmemControl) + mul_size(sizeof(PgStat_IO), NumProcStatSlots)); sz = add_size(sz, pgstat_dsa_init_size()); /* Add shared memory for all the custom fixed-numbered statistics */ @@ -172,7 +172,7 @@ StatsShmemInit(void) Assert(!found); /* the allocation of pgStatLocal.shmem itself */ - p += MAXALIGN(sizeof(PgStat_ShmemControl)); + p += MAXALIGN(sizeof(PgStat_ShmemControl) + mul_size(sizeof(PgStat_IO), NumProcStatSlots)); /* * Create a small dsa allocation in plain shared memory. This is diff --git a/src/backend/utils/adt/pgstatfuncs.c b/src/backend/utils/adt/pgstatfuncs.c index 3221137123..58e321f421 100644 --- a/src/backend/utils/adt/pgstatfuncs.c +++ b/src/backend/utils/adt/pgstatfuncs.c @@ -1359,18 +1359,30 @@ pg_stat_get_io(PG_FUNCTION_ARGS) ReturnSetInfo *rsinfo; PgStat_IO *backends_io_stats; Datum reset_time; + bool is_global; + + is_global = PG_GETARG_BOOL(0); InitMaterializedSRF(fcinfo, 0); rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; - backends_io_stats = pgstat_fetch_stat_io(); + if (is_global) + backends_io_stats = pgstat_fetch_global_stat_io(); + else + backends_io_stats = pgstat_fetch_my_stat_io(); reset_time = TimestampTzGetDatum(backends_io_stats->stat_reset_timestamp); for (int bktype = 0; bktype < BACKEND_NUM_TYPES; bktype++) { - Datum bktype_desc = CStringGetTextDatum(GetBackendTypeDesc(bktype)); - PgStat_BktypeIO *bktype_stats = &backends_io_stats->stats[bktype]; + Datum bktype_desc; + PgStat_BktypeIO *bktype_stats; + + if (!is_global && bktype != MyBackendType) + continue; + + bktype_desc = CStringGetTextDatum(GetBackendTypeDesc(bktype)); + bktype_stats = &backends_io_stats->stats[bktype]; /* * In Assert builds, we can afford an extra loop through all of the diff --git a/src/include/catalog/catversion.h b/src/include/catalog/catversion.h index 1980d492c3..fbee0db2eb 100644 --- a/src/include/catalog/catversion.h +++ b/src/include/catalog/catversion.h @@ -57,6 +57,6 @@ */ /* yyyymmddN */ -#define CATALOG_VERSION_NO 202408301 +#define CATALOG_VERSION_NO 202409021 #endif diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index 85f42be1b3..1184341fc8 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -5825,10 +5825,10 @@ { oid => '6214', descr => 'statistics: per backend type IO statistics', proname => 'pg_stat_get_io', prorows => '30', proretset => 't', provolatile => 'v', proparallel => 'r', prorettype => 'record', - proargtypes => '', - proallargtypes => '{text,text,text,int8,float8,int8,float8,int8,float8,int8,float8,int8,int8,int8,int8,int8,float8,timestamptz}', - proargmodes => '{o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o}', - proargnames => '{backend_type,object,context,reads,read_time,writes,write_time,writebacks,writeback_time,extends,extend_time,op_bytes,hits,evictions,reuses,fsyncs,fsync_time,stats_reset}', + proargtypes => 'bool', + proallargtypes => '{bool,text,text,text,int8,float8,int8,float8,int8,float8,int8,float8,int8,int8,int8,int8,int8,float8,timestamptz}', + proargmodes => '{i,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o}', + proargnames => '{is_global,backend_type,object,context,reads,read_time,writes,write_time,writebacks,writeback_time,extends,extend_time,op_bytes,hits,evictions,reuses,fsyncs,fsync_time,stats_reset}', prosrc => 'pg_stat_get_io' }, { oid => '1136', descr => 'statistics: information about WAL activity', diff --git a/src/include/pgstat.h b/src/include/pgstat.h index f63159c55c..9ec4053a3e 100644 --- a/src/include/pgstat.h +++ b/src/include/pgstat.h @@ -15,6 +15,7 @@ #include "datatype/timestamp.h" #include "portability/instr_time.h" #include "postmaster/pgarch.h" /* for MAX_XFN_CHARS */ +#include "storage/proc.h" #include "utils/backend_progress.h" /* for backward compatibility */ #include "utils/backend_status.h" /* for backward compatibility */ #include "utils/relcache.h" @@ -75,6 +76,8 @@ */ #define PGSTAT_KIND_EXPERIMENTAL 128 +#define NumProcStatSlots (MaxBackends + NUM_AUXILIARY_PROCS) + static inline bool pgstat_is_kind_builtin(PgStat_Kind kind) { @@ -556,7 +559,8 @@ extern instr_time pgstat_prepare_io_time(bool track_io_guc); extern void pgstat_count_io_op_time(IOObject io_object, IOContext io_context, IOOp io_op, instr_time start_time, uint32 cnt); -extern PgStat_IO *pgstat_fetch_stat_io(void); +extern PgStat_IO *pgstat_fetch_global_stat_io(void); +extern PgStat_IO *pgstat_fetch_my_stat_io(void); extern const char *pgstat_get_io_context_name(IOContext io_context); extern const char *pgstat_get_io_object_name(IOObject io_object); @@ -565,7 +569,7 @@ extern bool pgstat_tracks_io_object(BackendType bktype, IOObject io_object, IOContext io_context); extern bool pgstat_tracks_io_op(BackendType bktype, IOObject io_object, IOContext io_context, IOOp io_op); - +extern void pgstat_reset_io_counter_internal(int index, TimestampTz ts); /* * Functions in pgstat_database.c diff --git a/src/include/utils/pgstat_internal.h b/src/include/utils/pgstat_internal.h index fb132e439d..1eeb8aa107 100644 --- a/src/include/utils/pgstat_internal.h +++ b/src/include/utils/pgstat_internal.h @@ -347,11 +347,12 @@ typedef struct PgStatShared_Checkpointer typedef struct PgStatShared_IO { /* - * locks[i] protects stats.stats[i]. locks[0] also protects - * stats.stat_reset_timestamp. + * locks[i] protects global_stats.stats[i]. locks[0] also protects + * global_stats.stat_reset_timestamp. */ LWLock locks[BACKEND_NUM_TYPES]; - PgStat_IO stats; + PgStat_IO global_stats; + PgStat_IO stats[FLEXIBLE_ARRAY_MEMBER]; } PgStatShared_IO; typedef struct PgStatShared_SLRU @@ -444,7 +445,6 @@ typedef struct PgStat_ShmemControl PgStatShared_Archiver archiver; PgStatShared_BgWriter bgwriter; PgStatShared_Checkpointer checkpointer; - PgStatShared_IO io; PgStatShared_SLRU slru; PgStatShared_Wal wal; @@ -454,6 +454,8 @@ typedef struct PgStat_ShmemControl */ void *custom_data[PGSTAT_KIND_CUSTOM_SIZE]; + /* has to be at the end due to FLEXIBLE_ARRAY_MEMBER */ + PgStatShared_IO io; } PgStat_ShmemControl; @@ -475,7 +477,8 @@ typedef struct PgStat_Snapshot PgStat_CheckpointerStats checkpointer; - PgStat_IO io; + PgStat_IO my_io; + PgStat_IO global_io; PgStat_SLRUStats slru[SLRU_NUM_ELEMENTS]; diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out index 862433ee52..30dcb80f7b 100644 --- a/src/test/regress/expected/rules.out +++ b/src/test/regress/expected/rules.out @@ -1398,6 +1398,25 @@ pg_matviews| SELECT n.nspname AS schemaname, LEFT JOIN pg_namespace n ON ((n.oid = c.relnamespace))) LEFT JOIN pg_tablespace t ON ((t.oid = c.reltablespace))) WHERE (c.relkind = 'm'::"char"); +pg_my_stat_io| SELECT backend_type, + object, + context, + reads, + read_time, + writes, + write_time, + writebacks, + writeback_time, + extends, + extend_time, + op_bytes, + hits, + evictions, + reuses, + fsyncs, + fsync_time, + stats_reset + FROM pg_stat_get_io(false) b(backend_type, object, context, reads, read_time, writes, write_time, writebacks, writeback_time, extends, extend_time, op_bytes, hits, evictions, reuses, fsyncs, fsync_time, stats_reset); pg_policies| SELECT n.nspname AS schemaname, c.relname AS tablename, pol.polname AS policyname, @@ -1902,7 +1921,7 @@ pg_stat_io| SELECT backend_type, fsyncs, fsync_time, stats_reset - FROM pg_stat_get_io() b(backend_type, object, context, reads, read_time, writes, write_time, writebacks, writeback_time, extends, extend_time, op_bytes, hits, evictions, reuses, fsyncs, fsync_time, stats_reset); + FROM pg_stat_get_io(true) b(backend_type, object, context, reads, read_time, writes, write_time, writebacks, writeback_time, extends, extend_time, op_bytes, hits, evictions, reuses, fsyncs, fsync_time, stats_reset); pg_stat_progress_analyze| SELECT s.pid, s.datid, d.datname, diff --git a/src/test/regress/expected/stats.out b/src/test/regress/expected/stats.out index 6e08898b18..c489e528e0 100644 --- a/src/test/regress/expected/stats.out +++ b/src/test/regress/expected/stats.out @@ -1249,7 +1249,7 @@ SELECT pg_stat_get_subscription_stats(NULL); (1 row) --- Test that the following operations are tracked in pg_stat_io: +-- Test that the following operations are tracked in pg_[my]_stat_io: -- - reads of target blocks into shared buffers -- - writes of shared buffers to permanent storage -- - extends of relations using shared buffers @@ -1261,9 +1261,14 @@ SELECT pg_stat_get_subscription_stats(NULL); -- extends. SELECT sum(extends) AS io_sum_shared_before_extends FROM pg_stat_io WHERE context = 'normal' AND object = 'relation' \gset +SELECT sum(extends) AS my_io_sum_shared_before_extends + FROM pg_my_stat_io WHERE context = 'normal' AND object = 'relation' \gset SELECT sum(writes) AS writes, sum(fsyncs) AS fsyncs FROM pg_stat_io WHERE object = 'relation' \gset io_sum_shared_before_ +SELECT sum(writes) AS writes, sum(fsyncs) AS fsyncs + FROM pg_my_stat_io + WHERE object = 'relation' \gset my_io_sum_shared_before_ CREATE TABLE test_io_shared(a int); INSERT INTO test_io_shared SELECT i FROM generate_series(1,100)i; SELECT pg_stat_force_next_flush(); @@ -1280,8 +1285,16 @@ SELECT :io_sum_shared_after_extends > :io_sum_shared_before_extends; t (1 row) +SELECT sum(extends) AS my_io_sum_shared_after_extends + FROM pg_my_stat_io WHERE context = 'normal' AND object = 'relation' \gset +SELECT :my_io_sum_shared_after_extends > :my_io_sum_shared_before_extends; + ?column? +---------- + t +(1 row) + -- After a checkpoint, there should be some additional IOCONTEXT_NORMAL writes --- and fsyncs. +-- and fsyncs in the global stats (not for the backend). -- See comment above for rationale for two explicit CHECKPOINTs. CHECKPOINT; CHECKPOINT; @@ -1301,6 +1314,23 @@ SELECT current_setting('fsync') = 'off' t (1 row) +SELECT sum(writes) AS writes, sum(fsyncs) AS fsyncs + FROM pg_my_stat_io + WHERE object = 'relation' \gset my_io_sum_shared_after_ +SELECT :my_io_sum_shared_after_writes >= :my_io_sum_shared_before_writes; + ?column? +---------- + t +(1 row) + +SELECT current_setting('fsync') = 'off' + OR (:my_io_sum_shared_after_fsyncs = :my_io_sum_shared_before_fsyncs + AND :my_io_sum_shared_after_fsyncs= 0); + ?column? +---------- + t +(1 row) + -- Change the tablespace so that the table is rewritten directly, then SELECT -- from it to cause it to be read back into shared buffers. SELECT sum(reads) AS io_sum_shared_before_reads @@ -1521,6 +1551,8 @@ SELECT pg_stat_have_stats('io', 0, 0); SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS io_stats_pre_reset FROM pg_stat_io \gset +SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS my_io_stats_pre_reset + FROM pg_my_stat_io \gset SELECT pg_stat_reset_shared('io'); pg_stat_reset_shared ---------------------- @@ -1535,6 +1567,14 @@ SELECT :io_stats_post_reset < :io_stats_pre_reset; t (1 row) +SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS my_io_stats_post_reset + FROM pg_my_stat_io \gset +SELECT :my_io_stats_post_reset < :my_io_stats_pre_reset; + ?column? +---------- + t +(1 row) + -- test BRIN index doesn't block HOT update CREATE TABLE brin_hot ( id integer PRIMARY KEY, diff --git a/src/test/regress/sql/stats.sql b/src/test/regress/sql/stats.sql index d8ac0d06f4..c95cb71652 100644 --- a/src/test/regress/sql/stats.sql +++ b/src/test/regress/sql/stats.sql @@ -595,7 +595,7 @@ SELECT pg_stat_get_replication_slot(NULL); SELECT pg_stat_get_subscription_stats(NULL); --- Test that the following operations are tracked in pg_stat_io: +-- Test that the following operations are tracked in pg_[my]_stat_io: -- - reads of target blocks into shared buffers -- - writes of shared buffers to permanent storage -- - extends of relations using shared buffers @@ -609,18 +609,26 @@ SELECT pg_stat_get_subscription_stats(NULL); -- extends. SELECT sum(extends) AS io_sum_shared_before_extends FROM pg_stat_io WHERE context = 'normal' AND object = 'relation' \gset +SELECT sum(extends) AS my_io_sum_shared_before_extends + FROM pg_my_stat_io WHERE context = 'normal' AND object = 'relation' \gset SELECT sum(writes) AS writes, sum(fsyncs) AS fsyncs FROM pg_stat_io WHERE object = 'relation' \gset io_sum_shared_before_ +SELECT sum(writes) AS writes, sum(fsyncs) AS fsyncs + FROM pg_my_stat_io + WHERE object = 'relation' \gset my_io_sum_shared_before_ CREATE TABLE test_io_shared(a int); INSERT INTO test_io_shared SELECT i FROM generate_series(1,100)i; SELECT pg_stat_force_next_flush(); SELECT sum(extends) AS io_sum_shared_after_extends FROM pg_stat_io WHERE context = 'normal' AND object = 'relation' \gset SELECT :io_sum_shared_after_extends > :io_sum_shared_before_extends; +SELECT sum(extends) AS my_io_sum_shared_after_extends + FROM pg_my_stat_io WHERE context = 'normal' AND object = 'relation' \gset +SELECT :my_io_sum_shared_after_extends > :my_io_sum_shared_before_extends; -- After a checkpoint, there should be some additional IOCONTEXT_NORMAL writes --- and fsyncs. +-- and fsyncs in the global stats (not for the backend). -- See comment above for rationale for two explicit CHECKPOINTs. CHECKPOINT; CHECKPOINT; @@ -630,7 +638,13 @@ SELECT sum(writes) AS writes, sum(fsyncs) AS fsyncs SELECT :io_sum_shared_after_writes > :io_sum_shared_before_writes; SELECT current_setting('fsync') = 'off' OR :io_sum_shared_after_fsyncs > :io_sum_shared_before_fsyncs; - +SELECT sum(writes) AS writes, sum(fsyncs) AS fsyncs + FROM pg_my_stat_io + WHERE object = 'relation' \gset my_io_sum_shared_after_ +SELECT :my_io_sum_shared_after_writes >= :my_io_sum_shared_before_writes; +SELECT current_setting('fsync') = 'off' + OR (:my_io_sum_shared_after_fsyncs = :my_io_sum_shared_before_fsyncs + AND :my_io_sum_shared_after_fsyncs= 0); -- Change the tablespace so that the table is rewritten directly, then SELECT -- from it to cause it to be read back into shared buffers. SELECT sum(reads) AS io_sum_shared_before_reads @@ -762,10 +776,15 @@ SELECT :io_sum_bulkwrite_strategy_extends_after > :io_sum_bulkwrite_strategy_ext SELECT pg_stat_have_stats('io', 0, 0); SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS io_stats_pre_reset FROM pg_stat_io \gset +SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS my_io_stats_pre_reset + FROM pg_my_stat_io \gset SELECT pg_stat_reset_shared('io'); SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS io_stats_post_reset FROM pg_stat_io \gset SELECT :io_stats_post_reset < :io_stats_pre_reset; +SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS my_io_stats_post_reset + FROM pg_my_stat_io \gset +SELECT :my_io_stats_post_reset < :my_io_stats_pre_reset; -- test BRIN index doesn't block HOT update -- 2.34.1 --8G6ZM24Fzj1jtUgQ Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v1-0002-Add-pg_stat_get_backend_io.patch" ^ permalink raw reply [nested|flat] 25+ messages in thread
* [PATCH v2 1/2] per backend I/O statistics @ 2024-08-22 15:16 Bertrand Drouvot <[email protected]> 0 siblings, 0 replies; 25+ messages in thread From: Bertrand Drouvot @ 2024-08-22 15:16 UTC (permalink / raw) While pg_stat_io provides cluster-wide I/O statistics, this commit adds a new pg_my_stat_io view to display "my" backend I/O statistics. The KIND_IO stats are still "fixed amount" ones as the maximum number of backend is fixed. The statistics snapshot is made for the global stats (the aggregated ones) and for my backend stats. The snapshot is not build in each backend to copy all the others backends stats, as 1/ there is no use case (there is no need to get others backends I/O statistics while taking care of the stats_fetch_consistency) and 2/ that could be memory expensive depending of the number of max connections. A subsequent commit will add a new pg_stat_get_backend_io() function to be able to retrieve the I/O statistics for a given backend pid (each execution re-fetches counters from shared memory because as stated above there is no snapshots being created in each backend to copy the other backends stats). Bump catalog version needs to be done. --- doc/src/sgml/config.sgml | 4 +- doc/src/sgml/monitoring.sgml | 28 ++++++ src/backend/catalog/system_views.sql | 22 +++++ src/backend/utils/activity/pgstat.c | 11 ++- src/backend/utils/activity/pgstat_io.c | 91 ++++++++++++++--- src/backend/utils/activity/pgstat_shmem.c | 4 +- src/backend/utils/adt/pgstatfuncs.c | 114 +++++++++++++++++++++- src/include/catalog/pg_proc.dat | 9 ++ src/include/pgstat.h | 14 ++- src/include/utils/pgstat_internal.h | 14 ++- src/test/regress/expected/rules.out | 19 ++++ src/test/regress/expected/stats.out | 44 ++++++++- src/test/regress/sql/stats.sql | 25 ++++- 13 files changed, 366 insertions(+), 33 deletions(-) 8.5% doc/src/sgml/ 29.4% src/backend/utils/activity/ 23.9% src/backend/utils/adt/ 4.4% src/include/catalog/ 3.1% src/include/utils/ 3.1% src/include/ 14.4% src/test/regress/expected/ 10.0% src/test/regress/sql/ diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 0aec11f443..2a59b97093 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -8331,7 +8331,9 @@ COPY postgres_log FROM '/full/path/to/logfile.csv' WITH csv; displayed in <link linkend="monitoring-pg-stat-database-view"> <structname>pg_stat_database</structname></link>, <link linkend="monitoring-pg-stat-io-view"> - <structname>pg_stat_io</structname></link>, in the output of + <structname>pg_stat_io</structname></link>, + <link linkend="monitoring-pg-my-stat-io-view"> + <structname>pg_my_stat_io</structname></link>, in the output of <xref linkend="sql-explain"/> when the <literal>BUFFERS</literal> option is used, in the output of <xref linkend="sql-vacuum"/> when the <literal>VERBOSE</literal> option is used, by autovacuum diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml index 933de6fe07..b28ca4e0ca 100644 --- a/doc/src/sgml/monitoring.sgml +++ b/doc/src/sgml/monitoring.sgml @@ -488,6 +488,16 @@ postgres 27093 0.0 0.0 30096 2752 ? Ss 11:34 0:00 postgres: ser </entry> </row> + <row> + <entry><structname>pg_my_stat_io</structname><indexterm><primary>pg_my_stat_io</primary></indexterm></entry> + <entry> + One row for each combination of context and target object containing + my backend I/O statistics. + See <link linkend="monitoring-pg-my-stat-io-view"> + <structname>pg_my_stat_io</structname></link> for details. + </entry> + </row> + <row> <entry><structname>pg_stat_replication_slots</structname><indexterm><primary>pg_stat_replication_slots</primary></indexterm></entry> <entry>One row per replication slot, showing statistics about the @@ -2948,6 +2958,24 @@ description | Waiting for a newly initialized WAL file to reach durable storage + </sect2> + + <sect2 id="monitoring-pg-my-stat-io-view"> + <title><structname>pg_my_stat_io</structname></title> + + <indexterm> + <primary>pg_my_stat_io</primary> + </indexterm> + + <para> + The <structname>pg_my_stat_io</structname> view will contain one row for each + combination of target I/O object and I/O context, showing + my backend I/O statistics. Combinations which do not make sense are + omitted. The fields are exactly the same as the ones in the <link + linkend="monitoring-pg-stat-io-view"> <structname>pg_stat_io</structname></link> + view. + </para> + </sect2> <sect2 id="monitoring-pg-stat-bgwriter-view"> diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index 7fd5d256a1..b4939b7bc6 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1168,6 +1168,28 @@ SELECT b.stats_reset FROM pg_stat_get_io() b; +CREATE VIEW pg_my_stat_io AS +SELECT + b.backend_type, + b.object, + b.context, + b.reads, + b.read_time, + b.writes, + b.write_time, + b.writebacks, + b.writeback_time, + b.extends, + b.extend_time, + b.op_bytes, + b.hits, + b.evictions, + b.reuses, + b.fsyncs, + b.fsync_time, + b.stats_reset +FROM pg_stat_get_my_io() b; + CREATE VIEW pg_stat_wal AS SELECT w.wal_records, diff --git a/src/backend/utils/activity/pgstat.c b/src/backend/utils/activity/pgstat.c index 178b5ef65a..53b6e0bcd5 100644 --- a/src/backend/utils/activity/pgstat.c +++ b/src/backend/utils/activity/pgstat.c @@ -406,10 +406,12 @@ static const PgStat_KindInfo pgstat_kind_builtin_infos[PGSTAT_KIND_BUILTIN_SIZE] .fixed_amount = true, - .snapshot_ctl_off = offsetof(PgStat_Snapshot, io), + .init_backend_cb = pgstat_io_init_backend_cb, + .snapshot_ctl_off = offsetof(PgStat_Snapshot, global_io), .shared_ctl_off = offsetof(PgStat_ShmemControl, io), - .shared_data_off = offsetof(PgStatShared_IO, stats), - .shared_data_len = sizeof(((PgStatShared_IO *) 0)->stats), + /* [de-]serialize global_stats only */ + .shared_data_off = offsetof(PgStatShared_IO, global_stats), + .shared_data_len = sizeof(((PgStatShared_IO *) 0)->global_stats), .init_shmem_cb = pgstat_io_init_shmem_cb, .reset_all_cb = pgstat_io_reset_all_cb, @@ -581,6 +583,9 @@ pgstat_shutdown_hook(int code, Datum arg) pgstat_report_stat(true); + /* reset the pgstat_io counter related to this proc number */ + pgstat_reset_io_counter_internal(MyProcNumber, GetCurrentTimestamp()); + /* there shouldn't be any pending changes left */ Assert(dlist_is_empty(&pgStatPending)); dlist_init(&pgStatPending); diff --git a/src/backend/utils/activity/pgstat_io.c b/src/backend/utils/activity/pgstat_io.c index 8af55989ee..f2b36d8efa 100644 --- a/src/backend/utils/activity/pgstat_io.c +++ b/src/backend/utils/activity/pgstat_io.c @@ -31,6 +31,7 @@ typedef struct PgStat_PendingIO static PgStat_PendingIO PendingIOStats; bool have_iostats = false; +void pgstat_reset_io_counter_internal(int index, TimestampTz ts); /* * Check that stats have not been counted for any combination of IOObject, @@ -154,11 +155,19 @@ pgstat_count_io_op_time(IOObject io_object, IOContext io_context, IOOp io_op, } PgStat_IO * -pgstat_fetch_stat_io(void) +pgstat_fetch_global_stat_io(void) { pgstat_snapshot_fixed(PGSTAT_KIND_IO); - return &pgStatLocal.snapshot.io; + return &pgStatLocal.snapshot.global_io; +} + +PgStat_Backend_IO * +pgstat_fetch_my_stat_io(void) +{ + pgstat_snapshot_fixed(PGSTAT_KIND_IO); + + return &pgStatLocal.snapshot.my_io; } /* @@ -174,13 +183,16 @@ pgstat_flush_io(bool nowait) { LWLock *bktype_lock; PgStat_BktypeIO *bktype_shstats; + PgStat_BktypeIO *global_bktype_shstats; if (!have_iostats) return false; bktype_lock = &pgStatLocal.shmem->io.locks[MyBackendType]; bktype_shstats = - &pgStatLocal.shmem->io.stats.stats[MyBackendType]; + &pgStatLocal.shmem->io.stat[MyProcNumber].stats; + global_bktype_shstats = + &pgStatLocal.shmem->io.global_stats.stats[MyBackendType]; if (!nowait) LWLockAcquire(bktype_lock, LW_EXCLUSIVE); @@ -194,19 +206,28 @@ pgstat_flush_io(bool nowait) for (int io_op = 0; io_op < IOOP_NUM_TYPES; io_op++) { instr_time time; + PgStat_Counter counter; + + counter = PendingIOStats.counts[io_object][io_context][io_op]; - bktype_shstats->counts[io_object][io_context][io_op] += - PendingIOStats.counts[io_object][io_context][io_op]; + bktype_shstats->counts[io_object][io_context][io_op] += counter; + + global_bktype_shstats->counts[io_object][io_context][io_op] += + counter; time = PendingIOStats.pending_times[io_object][io_context][io_op]; bktype_shstats->times[io_object][io_context][io_op] += INSTR_TIME_GET_MICROSEC(time); + + global_bktype_shstats->times[io_object][io_context][io_op] += + INSTR_TIME_GET_MICROSEC(time); } } } Assert(pgstat_bktype_io_stats_valid(bktype_shstats, MyBackendType)); + Assert(pgstat_bktype_io_stats_valid(global_bktype_shstats, MyBackendType)); LWLockRelease(bktype_lock); @@ -260,13 +281,35 @@ pgstat_io_init_shmem_cb(void *stats) LWLockInitialize(&stat_shmem->locks[i], LWTRANCHE_PGSTATS_DATA); } +void +pgstat_reset_io_counter_internal(int index, TimestampTz ts) +{ + LWLock *bktype_lock = &pgStatLocal.shmem->io.locks[0]; + PgStat_BktypeIO *bktype_shstats = &pgStatLocal.shmem->io.stat[index].stats; + + + /* + * Use the lock in the first BackendType's PgStat_BktypeIO to protect the + * reset timestamp as well. + */ + LWLockAcquire(bktype_lock, LW_EXCLUSIVE); + + pgStatLocal.shmem->io.stat[index].stat_reset_timestamp = ts; + + memset(bktype_shstats, 0, sizeof(*bktype_shstats)); + LWLockRelease(bktype_lock); +} + void pgstat_io_reset_all_cb(TimestampTz ts) { + for (int i = 0; i < NumProcStatSlots; i++) + pgstat_reset_io_counter_internal(i, ts); + for (int i = 0; i < BACKEND_NUM_TYPES; i++) { LWLock *bktype_lock = &pgStatLocal.shmem->io.locks[i]; - PgStat_BktypeIO *bktype_shstats = &pgStatLocal.shmem->io.stats.stats[i]; + PgStat_BktypeIO *bktype_shstats = &pgStatLocal.shmem->io.global_stats.stats[i]; LWLockAcquire(bktype_lock, LW_EXCLUSIVE); @@ -275,7 +318,7 @@ pgstat_io_reset_all_cb(TimestampTz ts) * the reset timestamp as well. */ if (i == 0) - pgStatLocal.shmem->io.stats.stat_reset_timestamp = ts; + pgStatLocal.shmem->io.global_stats.stat_reset_timestamp = ts; memset(bktype_shstats, 0, sizeof(*bktype_shstats)); LWLockRelease(bktype_lock); @@ -285,11 +328,28 @@ pgstat_io_reset_all_cb(TimestampTz ts) void pgstat_io_snapshot_cb(void) { + /* First, our own stats */ + PgStat_BktypeIO *bktype_shstats = &pgStatLocal.shmem->io.stat[MyProcNumber].stats; + PgStat_BktypeIO *bktype_snap = &pgStatLocal.snapshot.my_io.stats; + LWLock *mybktype_lock = &pgStatLocal.shmem->io.locks[0]; + + LWLockAcquire(mybktype_lock, LW_SHARED); + + pgStatLocal.snapshot.my_io.stat_reset_timestamp = + pgStatLocal.shmem->io.stat[MyProcNumber].stat_reset_timestamp; + + pgStatLocal.snapshot.my_io.bktype = + pgStatLocal.shmem->io.stat[MyProcNumber].bktype; + + *bktype_snap = *bktype_shstats; + LWLockRelease(mybktype_lock); + + /* Now, the global stats */ for (int i = 0; i < BACKEND_NUM_TYPES; i++) { LWLock *bktype_lock = &pgStatLocal.shmem->io.locks[i]; - PgStat_BktypeIO *bktype_shstats = &pgStatLocal.shmem->io.stats.stats[i]; - PgStat_BktypeIO *bktype_snap = &pgStatLocal.snapshot.io.stats[i]; + PgStat_BktypeIO *bktype_global_shstats = &pgStatLocal.shmem->io.global_stats.stats[i]; + PgStat_BktypeIO *bktype_global_snap = &pgStatLocal.snapshot.global_io.stats[i]; LWLockAcquire(bktype_lock, LW_SHARED); @@ -298,11 +358,11 @@ pgstat_io_snapshot_cb(void) * the reset timestamp as well. */ if (i == 0) - pgStatLocal.snapshot.io.stat_reset_timestamp = - pgStatLocal.shmem->io.stats.stat_reset_timestamp; + pgStatLocal.snapshot.global_io.stat_reset_timestamp = + pgStatLocal.shmem->io.global_stats.stat_reset_timestamp; /* using struct assignment due to better type safety */ - *bktype_snap = *bktype_shstats; + *bktype_global_snap = *bktype_global_shstats; LWLockRelease(bktype_lock); } } @@ -485,3 +545,10 @@ pgstat_tracks_io_op(BackendType bktype, IOObject io_object, return true; } + +void +pgstat_io_init_backend_cb(void) +{ + /* Initialize our backend type in the IO statistics */ + pgStatLocal.shmem->io.stat[MyProcNumber].bktype = MyBackendType; +} diff --git a/src/backend/utils/activity/pgstat_shmem.c b/src/backend/utils/activity/pgstat_shmem.c index ec93bf6902..3cfcba9609 100644 --- a/src/backend/utils/activity/pgstat_shmem.c +++ b/src/backend/utils/activity/pgstat_shmem.c @@ -128,7 +128,7 @@ StatsShmemSize(void) { Size sz; - sz = MAXALIGN(sizeof(PgStat_ShmemControl)); + sz = MAXALIGN(sizeof(PgStat_ShmemControl) + mul_size(sizeof(PgStat_Backend_IO), NumProcStatSlots)); sz = add_size(sz, pgstat_dsa_init_size()); /* Add shared memory for all the custom fixed-numbered statistics */ @@ -172,7 +172,7 @@ StatsShmemInit(void) Assert(!found); /* the allocation of pgStatLocal.shmem itself */ - p += MAXALIGN(sizeof(PgStat_ShmemControl)); + p += MAXALIGN(sizeof(PgStat_ShmemControl) + mul_size(sizeof(PgStat_Backend_IO), NumProcStatSlots)); /* * Create a small dsa allocation in plain shared memory. This is diff --git a/src/backend/utils/adt/pgstatfuncs.c b/src/backend/utils/adt/pgstatfuncs.c index 97dc09ac0d..e72dc5daa7 100644 --- a/src/backend/utils/adt/pgstatfuncs.c +++ b/src/backend/utils/adt/pgstatfuncs.c @@ -1363,14 +1363,17 @@ pg_stat_get_io(PG_FUNCTION_ARGS) InitMaterializedSRF(fcinfo, 0); rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; - backends_io_stats = pgstat_fetch_stat_io(); + backends_io_stats = pgstat_fetch_global_stat_io(); reset_time = TimestampTzGetDatum(backends_io_stats->stat_reset_timestamp); for (int bktype = 0; bktype < BACKEND_NUM_TYPES; bktype++) { - Datum bktype_desc = CStringGetTextDatum(GetBackendTypeDesc(bktype)); - PgStat_BktypeIO *bktype_stats = &backends_io_stats->stats[bktype]; + Datum bktype_desc; + PgStat_BktypeIO *bktype_stats; + + bktype_desc = CStringGetTextDatum(GetBackendTypeDesc(bktype)); + bktype_stats = &backends_io_stats->stats[bktype]; /* * In Assert builds, we can afford an extra loop through all of the @@ -1462,6 +1465,111 @@ pg_stat_get_io(PG_FUNCTION_ARGS) return (Datum) 0; } +Datum +pg_stat_get_my_io(PG_FUNCTION_ARGS) +{ + ReturnSetInfo *rsinfo; + PgStat_Backend_IO *backend_io_stats; + Datum bktype_desc; + PgStat_BktypeIO *bktype_stats; + BackendType bktype; + Datum reset_time; + + InitMaterializedSRF(fcinfo, 0); + rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; + + backend_io_stats = pgstat_fetch_my_stat_io(); + + bktype = backend_io_stats->bktype; + bktype_desc = CStringGetTextDatum(GetBackendTypeDesc(bktype)); + bktype_stats = &backend_io_stats->stats; + reset_time = TimestampTzGetDatum(backend_io_stats->stat_reset_timestamp); + + /* + * In Assert builds, we can afford an extra loop through all of the + * counters checking that only expected stats are non-zero, since it keeps + * the non-Assert code cleaner. + */ + Assert(pgstat_bktype_io_stats_valid(bktype_stats, bktype)); + + for (int io_obj = 0; io_obj < IOOBJECT_NUM_TYPES; io_obj++) + { + const char *obj_name = pgstat_get_io_object_name(io_obj); + + for (int io_context = 0; io_context < IOCONTEXT_NUM_TYPES; io_context++) + { + const char *context_name = pgstat_get_io_context_name(io_context); + + Datum values[IO_NUM_COLUMNS] = {0}; + bool nulls[IO_NUM_COLUMNS] = {0}; + + /* + * Some combinations of BackendType, IOObject, and IOContext are + * not valid for any type of IOOp. In such cases, omit the entire + * row from the view. + */ + if (!pgstat_tracks_io_object(bktype, io_obj, io_context)) + continue; + + values[IO_COL_BACKEND_TYPE] = bktype_desc; + values[IO_COL_CONTEXT] = CStringGetTextDatum(context_name); + values[IO_COL_OBJECT] = CStringGetTextDatum(obj_name); + if (backend_io_stats->stat_reset_timestamp != 0) + values[IO_COL_RESET_TIME] = reset_time; + else + nulls[IO_COL_RESET_TIME] = true; + + /* + * Hard-code this to the value of BLCKSZ for now. Future values + * could include XLOG_BLCKSZ, once WAL IO is tracked, and constant + * multipliers, once non-block-oriented IO (e.g. temporary file + * IO) is tracked. + */ + values[IO_COL_CONVERSION] = Int64GetDatum(BLCKSZ); + + for (int io_op = 0; io_op < IOOP_NUM_TYPES; io_op++) + { + int op_idx = pgstat_get_io_op_index(io_op); + int time_idx = pgstat_get_io_time_index(io_op); + + /* + * Some combinations of BackendType and IOOp, of IOContext and + * IOOp, and of IOObject and IOOp are not tracked. Set these + * cells in the view NULL. + */ + if (pgstat_tracks_io_op(bktype, io_obj, io_context, io_op)) + { + PgStat_Counter count = + bktype_stats->counts[io_obj][io_context][io_op]; + + values[op_idx] = Int64GetDatum(count); + } + else + nulls[op_idx] = true; + + /* not every operation is timed */ + if (time_idx == IO_COL_INVALID) + continue; + + if (!nulls[op_idx]) + { + PgStat_Counter time = + bktype_stats->times[io_obj][io_context][io_op]; + + values[time_idx] = Float8GetDatum(pg_stat_us_to_ms(time)); + } + else + nulls[time_idx] = true; + } + + tuplestore_putvalues(rsinfo->setResult, rsinfo->setDesc, + values, nulls); + } + } + + return (Datum) 0; +} + /* * Returns statistics of WAL activity */ diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index ff5436acac..62658e9a00 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -5831,6 +5831,15 @@ proargnames => '{backend_type,object,context,reads,read_time,writes,write_time,writebacks,writeback_time,extends,extend_time,op_bytes,hits,evictions,reuses,fsyncs,fsync_time,stats_reset}', prosrc => 'pg_stat_get_io' }, +{ oid => '8806', descr => 'statistics: My backend IO statistics', + proname => 'pg_stat_get_my_io', prorows => '5', proretset => 't', + provolatile => 'v', proparallel => 'r', prorettype => 'record', + proargtypes => '', + proallargtypes => '{text,text,text,int8,float8,int8,float8,int8,float8,int8,float8,int8,int8,int8,int8,int8,float8,timestamptz}', + proargmodes => '{o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o}', + proargnames => '{backend_type,object,context,reads,read_time,writes,write_time,writebacks,writeback_time,extends,extend_time,op_bytes,hits,evictions,reuses,fsyncs,fsync_time,stats_reset}', + prosrc => 'pg_stat_get_my_io' }, + { oid => '1136', descr => 'statistics: information about WAL activity', proname => 'pg_stat_get_wal', proisstrict => 'f', provolatile => 's', proparallel => 'r', prorettype => 'record', proargtypes => '', diff --git a/src/include/pgstat.h b/src/include/pgstat.h index be2c91168a..9d63b1a5b7 100644 --- a/src/include/pgstat.h +++ b/src/include/pgstat.h @@ -16,6 +16,7 @@ #include "portability/instr_time.h" #include "postmaster/pgarch.h" /* for MAX_XFN_CHARS */ #include "replication/conflict.h" +#include "storage/proc.h" #include "utils/backend_progress.h" /* for backward compatibility */ #include "utils/backend_status.h" /* for backward compatibility */ #include "utils/relcache.h" @@ -76,6 +77,8 @@ */ #define PGSTAT_KIND_EXPERIMENTAL 128 +#define NumProcStatSlots (MaxBackends + NUM_AUXILIARY_PROCS) + static inline bool pgstat_is_kind_builtin(PgStat_Kind kind) { @@ -351,6 +354,12 @@ typedef struct PgStat_IO PgStat_BktypeIO stats[BACKEND_NUM_TYPES]; } PgStat_IO; +typedef struct PgStat_Backend_IO +{ + TimestampTz stat_reset_timestamp; + BackendType bktype; + PgStat_BktypeIO stats; +} PgStat_Backend_IO; typedef struct PgStat_StatDBEntry { @@ -559,7 +568,8 @@ extern instr_time pgstat_prepare_io_time(bool track_io_guc); extern void pgstat_count_io_op_time(IOObject io_object, IOContext io_context, IOOp io_op, instr_time start_time, uint32 cnt); -extern PgStat_IO *pgstat_fetch_stat_io(void); +extern PgStat_IO *pgstat_fetch_global_stat_io(void); +extern PgStat_Backend_IO *pgstat_fetch_my_stat_io(void); extern const char *pgstat_get_io_context_name(IOContext io_context); extern const char *pgstat_get_io_object_name(IOObject io_object); @@ -568,7 +578,7 @@ extern bool pgstat_tracks_io_object(BackendType bktype, IOObject io_object, IOContext io_context); extern bool pgstat_tracks_io_op(BackendType bktype, IOObject io_object, IOContext io_context, IOOp io_op); - +extern void pgstat_reset_io_counter_internal(int index, TimestampTz ts); /* * Functions in pgstat_database.c diff --git a/src/include/utils/pgstat_internal.h b/src/include/utils/pgstat_internal.h index 25820cbf0a..3b2f2b28dd 100644 --- a/src/include/utils/pgstat_internal.h +++ b/src/include/utils/pgstat_internal.h @@ -353,11 +353,12 @@ typedef struct PgStatShared_Checkpointer typedef struct PgStatShared_IO { /* - * locks[i] protects stats.stats[i]. locks[0] also protects - * stats.stat_reset_timestamp. + * locks[i] protects global_stats.stats[i]. locks[0] also protects + * global_stats.stat_reset_timestamp. */ LWLock locks[BACKEND_NUM_TYPES]; - PgStat_IO stats; + PgStat_IO global_stats; + PgStat_Backend_IO stat[FLEXIBLE_ARRAY_MEMBER]; } PgStatShared_IO; typedef struct PgStatShared_SLRU @@ -450,7 +451,6 @@ typedef struct PgStat_ShmemControl PgStatShared_Archiver archiver; PgStatShared_BgWriter bgwriter; PgStatShared_Checkpointer checkpointer; - PgStatShared_IO io; PgStatShared_SLRU slru; PgStatShared_Wal wal; @@ -460,6 +460,8 @@ typedef struct PgStat_ShmemControl */ void *custom_data[PGSTAT_KIND_CUSTOM_SIZE]; + /* has to be at the end due to FLEXIBLE_ARRAY_MEMBER */ + PgStatShared_IO io; } PgStat_ShmemControl; @@ -481,7 +483,8 @@ typedef struct PgStat_Snapshot PgStat_CheckpointerStats checkpointer; - PgStat_IO io; + PgStat_Backend_IO my_io; + PgStat_IO global_io; PgStat_SLRUStats slru[SLRU_NUM_ELEMENTS]; @@ -682,6 +685,7 @@ extern bool pgstat_flush_wal(bool nowait); extern bool pgstat_have_pending_wal(void); extern void pgstat_wal_init_backend_cb(void); +extern void pgstat_io_init_backend_cb(void); extern void pgstat_wal_init_shmem_cb(void *stats); extern void pgstat_wal_reset_all_cb(TimestampTz ts); extern void pgstat_wal_snapshot_cb(void); diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out index a1626f3fae..785298d7b0 100644 --- a/src/test/regress/expected/rules.out +++ b/src/test/regress/expected/rules.out @@ -1398,6 +1398,25 @@ pg_matviews| SELECT n.nspname AS schemaname, LEFT JOIN pg_namespace n ON ((n.oid = c.relnamespace))) LEFT JOIN pg_tablespace t ON ((t.oid = c.reltablespace))) WHERE (c.relkind = 'm'::"char"); +pg_my_stat_io| SELECT backend_type, + object, + context, + reads, + read_time, + writes, + write_time, + writebacks, + writeback_time, + extends, + extend_time, + op_bytes, + hits, + evictions, + reuses, + fsyncs, + fsync_time, + stats_reset + FROM pg_stat_get_my_io() b(backend_type, object, context, reads, read_time, writes, write_time, writebacks, writeback_time, extends, extend_time, op_bytes, hits, evictions, reuses, fsyncs, fsync_time, stats_reset); pg_policies| SELECT n.nspname AS schemaname, c.relname AS tablename, pol.polname AS policyname, diff --git a/src/test/regress/expected/stats.out b/src/test/regress/expected/stats.out index 6e08898b18..c489e528e0 100644 --- a/src/test/regress/expected/stats.out +++ b/src/test/regress/expected/stats.out @@ -1249,7 +1249,7 @@ SELECT pg_stat_get_subscription_stats(NULL); (1 row) --- Test that the following operations are tracked in pg_stat_io: +-- Test that the following operations are tracked in pg_[my]_stat_io: -- - reads of target blocks into shared buffers -- - writes of shared buffers to permanent storage -- - extends of relations using shared buffers @@ -1261,9 +1261,14 @@ SELECT pg_stat_get_subscription_stats(NULL); -- extends. SELECT sum(extends) AS io_sum_shared_before_extends FROM pg_stat_io WHERE context = 'normal' AND object = 'relation' \gset +SELECT sum(extends) AS my_io_sum_shared_before_extends + FROM pg_my_stat_io WHERE context = 'normal' AND object = 'relation' \gset SELECT sum(writes) AS writes, sum(fsyncs) AS fsyncs FROM pg_stat_io WHERE object = 'relation' \gset io_sum_shared_before_ +SELECT sum(writes) AS writes, sum(fsyncs) AS fsyncs + FROM pg_my_stat_io + WHERE object = 'relation' \gset my_io_sum_shared_before_ CREATE TABLE test_io_shared(a int); INSERT INTO test_io_shared SELECT i FROM generate_series(1,100)i; SELECT pg_stat_force_next_flush(); @@ -1280,8 +1285,16 @@ SELECT :io_sum_shared_after_extends > :io_sum_shared_before_extends; t (1 row) +SELECT sum(extends) AS my_io_sum_shared_after_extends + FROM pg_my_stat_io WHERE context = 'normal' AND object = 'relation' \gset +SELECT :my_io_sum_shared_after_extends > :my_io_sum_shared_before_extends; + ?column? +---------- + t +(1 row) + -- After a checkpoint, there should be some additional IOCONTEXT_NORMAL writes --- and fsyncs. +-- and fsyncs in the global stats (not for the backend). -- See comment above for rationale for two explicit CHECKPOINTs. CHECKPOINT; CHECKPOINT; @@ -1301,6 +1314,23 @@ SELECT current_setting('fsync') = 'off' t (1 row) +SELECT sum(writes) AS writes, sum(fsyncs) AS fsyncs + FROM pg_my_stat_io + WHERE object = 'relation' \gset my_io_sum_shared_after_ +SELECT :my_io_sum_shared_after_writes >= :my_io_sum_shared_before_writes; + ?column? +---------- + t +(1 row) + +SELECT current_setting('fsync') = 'off' + OR (:my_io_sum_shared_after_fsyncs = :my_io_sum_shared_before_fsyncs + AND :my_io_sum_shared_after_fsyncs= 0); + ?column? +---------- + t +(1 row) + -- Change the tablespace so that the table is rewritten directly, then SELECT -- from it to cause it to be read back into shared buffers. SELECT sum(reads) AS io_sum_shared_before_reads @@ -1521,6 +1551,8 @@ SELECT pg_stat_have_stats('io', 0, 0); SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS io_stats_pre_reset FROM pg_stat_io \gset +SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS my_io_stats_pre_reset + FROM pg_my_stat_io \gset SELECT pg_stat_reset_shared('io'); pg_stat_reset_shared ---------------------- @@ -1535,6 +1567,14 @@ SELECT :io_stats_post_reset < :io_stats_pre_reset; t (1 row) +SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS my_io_stats_post_reset + FROM pg_my_stat_io \gset +SELECT :my_io_stats_post_reset < :my_io_stats_pre_reset; + ?column? +---------- + t +(1 row) + -- test BRIN index doesn't block HOT update CREATE TABLE brin_hot ( id integer PRIMARY KEY, diff --git a/src/test/regress/sql/stats.sql b/src/test/regress/sql/stats.sql index d8ac0d06f4..c95cb71652 100644 --- a/src/test/regress/sql/stats.sql +++ b/src/test/regress/sql/stats.sql @@ -595,7 +595,7 @@ SELECT pg_stat_get_replication_slot(NULL); SELECT pg_stat_get_subscription_stats(NULL); --- Test that the following operations are tracked in pg_stat_io: +-- Test that the following operations are tracked in pg_[my]_stat_io: -- - reads of target blocks into shared buffers -- - writes of shared buffers to permanent storage -- - extends of relations using shared buffers @@ -609,18 +609,26 @@ SELECT pg_stat_get_subscription_stats(NULL); -- extends. SELECT sum(extends) AS io_sum_shared_before_extends FROM pg_stat_io WHERE context = 'normal' AND object = 'relation' \gset +SELECT sum(extends) AS my_io_sum_shared_before_extends + FROM pg_my_stat_io WHERE context = 'normal' AND object = 'relation' \gset SELECT sum(writes) AS writes, sum(fsyncs) AS fsyncs FROM pg_stat_io WHERE object = 'relation' \gset io_sum_shared_before_ +SELECT sum(writes) AS writes, sum(fsyncs) AS fsyncs + FROM pg_my_stat_io + WHERE object = 'relation' \gset my_io_sum_shared_before_ CREATE TABLE test_io_shared(a int); INSERT INTO test_io_shared SELECT i FROM generate_series(1,100)i; SELECT pg_stat_force_next_flush(); SELECT sum(extends) AS io_sum_shared_after_extends FROM pg_stat_io WHERE context = 'normal' AND object = 'relation' \gset SELECT :io_sum_shared_after_extends > :io_sum_shared_before_extends; +SELECT sum(extends) AS my_io_sum_shared_after_extends + FROM pg_my_stat_io WHERE context = 'normal' AND object = 'relation' \gset +SELECT :my_io_sum_shared_after_extends > :my_io_sum_shared_before_extends; -- After a checkpoint, there should be some additional IOCONTEXT_NORMAL writes --- and fsyncs. +-- and fsyncs in the global stats (not for the backend). -- See comment above for rationale for two explicit CHECKPOINTs. CHECKPOINT; CHECKPOINT; @@ -630,7 +638,13 @@ SELECT sum(writes) AS writes, sum(fsyncs) AS fsyncs SELECT :io_sum_shared_after_writes > :io_sum_shared_before_writes; SELECT current_setting('fsync') = 'off' OR :io_sum_shared_after_fsyncs > :io_sum_shared_before_fsyncs; - +SELECT sum(writes) AS writes, sum(fsyncs) AS fsyncs + FROM pg_my_stat_io + WHERE object = 'relation' \gset my_io_sum_shared_after_ +SELECT :my_io_sum_shared_after_writes >= :my_io_sum_shared_before_writes; +SELECT current_setting('fsync') = 'off' + OR (:my_io_sum_shared_after_fsyncs = :my_io_sum_shared_before_fsyncs + AND :my_io_sum_shared_after_fsyncs= 0); -- Change the tablespace so that the table is rewritten directly, then SELECT -- from it to cause it to be read back into shared buffers. SELECT sum(reads) AS io_sum_shared_before_reads @@ -762,10 +776,15 @@ SELECT :io_sum_bulkwrite_strategy_extends_after > :io_sum_bulkwrite_strategy_ext SELECT pg_stat_have_stats('io', 0, 0); SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS io_stats_pre_reset FROM pg_stat_io \gset +SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS my_io_stats_pre_reset + FROM pg_my_stat_io \gset SELECT pg_stat_reset_shared('io'); SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS io_stats_post_reset FROM pg_stat_io \gset SELECT :io_stats_post_reset < :io_stats_pre_reset; +SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS my_io_stats_post_reset + FROM pg_my_stat_io \gset +SELECT :my_io_stats_post_reset < :my_io_stats_pre_reset; -- test BRIN index doesn't block HOT update -- 2.34.1 --0+8wqzuq5PafKmdo Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v2-0002-Add-pg_stat_get_backend_io.patch" ^ permalink raw reply [nested|flat] 25+ messages in thread
* [PATCH v3 1/2] per backend I/O statistics @ 2024-08-22 15:16 Bertrand Drouvot <[email protected]> 0 siblings, 0 replies; 25+ messages in thread From: Bertrand Drouvot @ 2024-08-22 15:16 UTC (permalink / raw) While pg_stat_io provides cluster-wide I/O statistics, this commit adds a new pg_my_stat_io view to display "my" backend I/O statistics. The KIND_IO stats are still "fixed amount" ones as the maximum number of backend is fixed. The statistics snapshot is made for the global stats (the aggregated ones) and for my backend stats. The snapshot is not build in each backend to copy all the others backends stats, as 1/ there is no use case (there is no need to get others backends I/O statistics while taking care of the stats_fetch_consistency) and 2/ that could be memory expensive depending of the number of max connections. A subsequent commit will add a new pg_stat_get_backend_io() function to be able to retrieve the I/O statistics for a given backend pid (each execution re-fetches counters from shared memory because as stated above there is no snapshots being created in each backend to copy the other backends stats). Bump catalog version needs to be done. --- doc/src/sgml/config.sgml | 4 +- doc/src/sgml/monitoring.sgml | 28 ++++++ src/backend/catalog/system_views.sql | 22 +++++ src/backend/utils/activity/pgstat.c | 11 ++- src/backend/utils/activity/pgstat_io.c | 91 ++++++++++++++--- src/backend/utils/activity/pgstat_shmem.c | 4 +- src/backend/utils/adt/pgstatfuncs.c | 114 +++++++++++++++++++++- src/include/catalog/pg_proc.dat | 9 ++ src/include/pgstat.h | 14 ++- src/include/utils/pgstat_internal.h | 14 ++- src/test/regress/expected/rules.out | 19 ++++ src/test/regress/expected/stats.out | 44 ++++++++- src/test/regress/sql/stats.sql | 25 ++++- 13 files changed, 366 insertions(+), 33 deletions(-) 8.5% doc/src/sgml/ 29.4% src/backend/utils/activity/ 23.9% src/backend/utils/adt/ 4.4% src/include/catalog/ 3.1% src/include/utils/ 3.1% src/include/ 14.4% src/test/regress/expected/ 10.0% src/test/regress/sql/ diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 0aec11f443..2a59b97093 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -8331,7 +8331,9 @@ COPY postgres_log FROM '/full/path/to/logfile.csv' WITH csv; displayed in <link linkend="monitoring-pg-stat-database-view"> <structname>pg_stat_database</structname></link>, <link linkend="monitoring-pg-stat-io-view"> - <structname>pg_stat_io</structname></link>, in the output of + <structname>pg_stat_io</structname></link>, + <link linkend="monitoring-pg-my-stat-io-view"> + <structname>pg_my_stat_io</structname></link>, in the output of <xref linkend="sql-explain"/> when the <literal>BUFFERS</literal> option is used, in the output of <xref linkend="sql-vacuum"/> when the <literal>VERBOSE</literal> option is used, by autovacuum diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml index 933de6fe07..b28ca4e0ca 100644 --- a/doc/src/sgml/monitoring.sgml +++ b/doc/src/sgml/monitoring.sgml @@ -488,6 +488,16 @@ postgres 27093 0.0 0.0 30096 2752 ? Ss 11:34 0:00 postgres: ser </entry> </row> + <row> + <entry><structname>pg_my_stat_io</structname><indexterm><primary>pg_my_stat_io</primary></indexterm></entry> + <entry> + One row for each combination of context and target object containing + my backend I/O statistics. + See <link linkend="monitoring-pg-my-stat-io-view"> + <structname>pg_my_stat_io</structname></link> for details. + </entry> + </row> + <row> <entry><structname>pg_stat_replication_slots</structname><indexterm><primary>pg_stat_replication_slots</primary></indexterm></entry> <entry>One row per replication slot, showing statistics about the @@ -2948,6 +2958,24 @@ description | Waiting for a newly initialized WAL file to reach durable storage + </sect2> + + <sect2 id="monitoring-pg-my-stat-io-view"> + <title><structname>pg_my_stat_io</structname></title> + + <indexterm> + <primary>pg_my_stat_io</primary> + </indexterm> + + <para> + The <structname>pg_my_stat_io</structname> view will contain one row for each + combination of target I/O object and I/O context, showing + my backend I/O statistics. Combinations which do not make sense are + omitted. The fields are exactly the same as the ones in the <link + linkend="monitoring-pg-stat-io-view"> <structname>pg_stat_io</structname></link> + view. + </para> + </sect2> <sect2 id="monitoring-pg-stat-bgwriter-view"> diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index 7fd5d256a1..b4939b7bc6 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1168,6 +1168,28 @@ SELECT b.stats_reset FROM pg_stat_get_io() b; +CREATE VIEW pg_my_stat_io AS +SELECT + b.backend_type, + b.object, + b.context, + b.reads, + b.read_time, + b.writes, + b.write_time, + b.writebacks, + b.writeback_time, + b.extends, + b.extend_time, + b.op_bytes, + b.hits, + b.evictions, + b.reuses, + b.fsyncs, + b.fsync_time, + b.stats_reset +FROM pg_stat_get_my_io() b; + CREATE VIEW pg_stat_wal AS SELECT w.wal_records, diff --git a/src/backend/utils/activity/pgstat.c b/src/backend/utils/activity/pgstat.c index a7f2dfc744..deb30cd2fc 100644 --- a/src/backend/utils/activity/pgstat.c +++ b/src/backend/utils/activity/pgstat.c @@ -406,10 +406,12 @@ static const PgStat_KindInfo pgstat_kind_builtin_infos[PGSTAT_KIND_BUILTIN_SIZE] .fixed_amount = true, - .snapshot_ctl_off = offsetof(PgStat_Snapshot, io), + .init_backend_cb = pgstat_io_init_backend_cb, + .snapshot_ctl_off = offsetof(PgStat_Snapshot, global_io), .shared_ctl_off = offsetof(PgStat_ShmemControl, io), - .shared_data_off = offsetof(PgStatShared_IO, stats), - .shared_data_len = sizeof(((PgStatShared_IO *) 0)->stats), + /* [de-]serialize global_stats only */ + .shared_data_off = offsetof(PgStatShared_IO, global_stats), + .shared_data_len = sizeof(((PgStatShared_IO *) 0)->global_stats), .flush_fixed_cb = pgstat_io_flush_cb, .have_fixed_pending_cb = pgstat_io_have_pending_cb, @@ -587,6 +589,9 @@ pgstat_shutdown_hook(int code, Datum arg) pgstat_report_stat(true); + /* reset the pgstat_io counter related to this proc number */ + pgstat_reset_io_counter_internal(MyProcNumber, GetCurrentTimestamp()); + /* there shouldn't be any pending changes left */ Assert(dlist_is_empty(&pgStatPending)); dlist_init(&pgStatPending); diff --git a/src/backend/utils/activity/pgstat_io.c b/src/backend/utils/activity/pgstat_io.c index cc2ffc78aa..611ca21720 100644 --- a/src/backend/utils/activity/pgstat_io.c +++ b/src/backend/utils/activity/pgstat_io.c @@ -31,6 +31,7 @@ typedef struct PgStat_PendingIO static PgStat_PendingIO PendingIOStats; static bool have_iostats = false; +void pgstat_reset_io_counter_internal(int index, TimestampTz ts); /* * Check that stats have not been counted for any combination of IOObject, @@ -154,11 +155,19 @@ pgstat_count_io_op_time(IOObject io_object, IOContext io_context, IOOp io_op, } PgStat_IO * -pgstat_fetch_stat_io(void) +pgstat_fetch_global_stat_io(void) { pgstat_snapshot_fixed(PGSTAT_KIND_IO); - return &pgStatLocal.snapshot.io; + return &pgStatLocal.snapshot.global_io; +} + +PgStat_Backend_IO * +pgstat_fetch_my_stat_io(void) +{ + pgstat_snapshot_fixed(PGSTAT_KIND_IO); + + return &pgStatLocal.snapshot.my_io; } /* @@ -192,13 +201,16 @@ pgstat_io_flush_cb(bool nowait) { LWLock *bktype_lock; PgStat_BktypeIO *bktype_shstats; + PgStat_BktypeIO *global_bktype_shstats; if (!have_iostats) return false; bktype_lock = &pgStatLocal.shmem->io.locks[MyBackendType]; bktype_shstats = - &pgStatLocal.shmem->io.stats.stats[MyBackendType]; + &pgStatLocal.shmem->io.stat[MyProcNumber].stats; + global_bktype_shstats = + &pgStatLocal.shmem->io.global_stats.stats[MyBackendType]; if (!nowait) LWLockAcquire(bktype_lock, LW_EXCLUSIVE); @@ -212,19 +224,28 @@ pgstat_io_flush_cb(bool nowait) for (int io_op = 0; io_op < IOOP_NUM_TYPES; io_op++) { instr_time time; + PgStat_Counter counter; + + counter = PendingIOStats.counts[io_object][io_context][io_op]; - bktype_shstats->counts[io_object][io_context][io_op] += - PendingIOStats.counts[io_object][io_context][io_op]; + bktype_shstats->counts[io_object][io_context][io_op] += counter; + + global_bktype_shstats->counts[io_object][io_context][io_op] += + counter; time = PendingIOStats.pending_times[io_object][io_context][io_op]; bktype_shstats->times[io_object][io_context][io_op] += INSTR_TIME_GET_MICROSEC(time); + + global_bktype_shstats->times[io_object][io_context][io_op] += + INSTR_TIME_GET_MICROSEC(time); } } } Assert(pgstat_bktype_io_stats_valid(bktype_shstats, MyBackendType)); + Assert(pgstat_bktype_io_stats_valid(global_bktype_shstats, MyBackendType)); LWLockRelease(bktype_lock); @@ -278,13 +299,35 @@ pgstat_io_init_shmem_cb(void *stats) LWLockInitialize(&stat_shmem->locks[i], LWTRANCHE_PGSTATS_DATA); } +void +pgstat_reset_io_counter_internal(int index, TimestampTz ts) +{ + LWLock *bktype_lock = &pgStatLocal.shmem->io.locks[0]; + PgStat_BktypeIO *bktype_shstats = &pgStatLocal.shmem->io.stat[index].stats; + + + /* + * Use the lock in the first BackendType's PgStat_BktypeIO to protect the + * reset timestamp as well. + */ + LWLockAcquire(bktype_lock, LW_EXCLUSIVE); + + pgStatLocal.shmem->io.stat[index].stat_reset_timestamp = ts; + + memset(bktype_shstats, 0, sizeof(*bktype_shstats)); + LWLockRelease(bktype_lock); +} + void pgstat_io_reset_all_cb(TimestampTz ts) { + for (int i = 0; i < NumProcStatSlots; i++) + pgstat_reset_io_counter_internal(i, ts); + for (int i = 0; i < BACKEND_NUM_TYPES; i++) { LWLock *bktype_lock = &pgStatLocal.shmem->io.locks[i]; - PgStat_BktypeIO *bktype_shstats = &pgStatLocal.shmem->io.stats.stats[i]; + PgStat_BktypeIO *bktype_shstats = &pgStatLocal.shmem->io.global_stats.stats[i]; LWLockAcquire(bktype_lock, LW_EXCLUSIVE); @@ -293,7 +336,7 @@ pgstat_io_reset_all_cb(TimestampTz ts) * the reset timestamp as well. */ if (i == 0) - pgStatLocal.shmem->io.stats.stat_reset_timestamp = ts; + pgStatLocal.shmem->io.global_stats.stat_reset_timestamp = ts; memset(bktype_shstats, 0, sizeof(*bktype_shstats)); LWLockRelease(bktype_lock); @@ -303,11 +346,28 @@ pgstat_io_reset_all_cb(TimestampTz ts) void pgstat_io_snapshot_cb(void) { + /* First, our own stats */ + PgStat_BktypeIO *bktype_shstats = &pgStatLocal.shmem->io.stat[MyProcNumber].stats; + PgStat_BktypeIO *bktype_snap = &pgStatLocal.snapshot.my_io.stats; + LWLock *mybktype_lock = &pgStatLocal.shmem->io.locks[0]; + + LWLockAcquire(mybktype_lock, LW_SHARED); + + pgStatLocal.snapshot.my_io.stat_reset_timestamp = + pgStatLocal.shmem->io.stat[MyProcNumber].stat_reset_timestamp; + + pgStatLocal.snapshot.my_io.bktype = + pgStatLocal.shmem->io.stat[MyProcNumber].bktype; + + *bktype_snap = *bktype_shstats; + LWLockRelease(mybktype_lock); + + /* Now, the global stats */ for (int i = 0; i < BACKEND_NUM_TYPES; i++) { LWLock *bktype_lock = &pgStatLocal.shmem->io.locks[i]; - PgStat_BktypeIO *bktype_shstats = &pgStatLocal.shmem->io.stats.stats[i]; - PgStat_BktypeIO *bktype_snap = &pgStatLocal.snapshot.io.stats[i]; + PgStat_BktypeIO *bktype_global_shstats = &pgStatLocal.shmem->io.global_stats.stats[i]; + PgStat_BktypeIO *bktype_global_snap = &pgStatLocal.snapshot.global_io.stats[i]; LWLockAcquire(bktype_lock, LW_SHARED); @@ -316,11 +376,11 @@ pgstat_io_snapshot_cb(void) * the reset timestamp as well. */ if (i == 0) - pgStatLocal.snapshot.io.stat_reset_timestamp = - pgStatLocal.shmem->io.stats.stat_reset_timestamp; + pgStatLocal.snapshot.global_io.stat_reset_timestamp = + pgStatLocal.shmem->io.global_stats.stat_reset_timestamp; /* using struct assignment due to better type safety */ - *bktype_snap = *bktype_shstats; + *bktype_global_snap = *bktype_global_shstats; LWLockRelease(bktype_lock); } } @@ -503,3 +563,10 @@ pgstat_tracks_io_op(BackendType bktype, IOObject io_object, return true; } + +void +pgstat_io_init_backend_cb(void) +{ + /* Initialize our backend type in the IO statistics */ + pgStatLocal.shmem->io.stat[MyProcNumber].bktype = MyBackendType; +} diff --git a/src/backend/utils/activity/pgstat_shmem.c b/src/backend/utils/activity/pgstat_shmem.c index ec93bf6902..3cfcba9609 100644 --- a/src/backend/utils/activity/pgstat_shmem.c +++ b/src/backend/utils/activity/pgstat_shmem.c @@ -128,7 +128,7 @@ StatsShmemSize(void) { Size sz; - sz = MAXALIGN(sizeof(PgStat_ShmemControl)); + sz = MAXALIGN(sizeof(PgStat_ShmemControl) + mul_size(sizeof(PgStat_Backend_IO), NumProcStatSlots)); sz = add_size(sz, pgstat_dsa_init_size()); /* Add shared memory for all the custom fixed-numbered statistics */ @@ -172,7 +172,7 @@ StatsShmemInit(void) Assert(!found); /* the allocation of pgStatLocal.shmem itself */ - p += MAXALIGN(sizeof(PgStat_ShmemControl)); + p += MAXALIGN(sizeof(PgStat_ShmemControl) + mul_size(sizeof(PgStat_Backend_IO), NumProcStatSlots)); /* * Create a small dsa allocation in plain shared memory. This is diff --git a/src/backend/utils/adt/pgstatfuncs.c b/src/backend/utils/adt/pgstatfuncs.c index 33c7b25560..4c00570396 100644 --- a/src/backend/utils/adt/pgstatfuncs.c +++ b/src/backend/utils/adt/pgstatfuncs.c @@ -1363,14 +1363,17 @@ pg_stat_get_io(PG_FUNCTION_ARGS) InitMaterializedSRF(fcinfo, 0); rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; - backends_io_stats = pgstat_fetch_stat_io(); + backends_io_stats = pgstat_fetch_global_stat_io(); reset_time = TimestampTzGetDatum(backends_io_stats->stat_reset_timestamp); for (int bktype = 0; bktype < BACKEND_NUM_TYPES; bktype++) { - Datum bktype_desc = CStringGetTextDatum(GetBackendTypeDesc(bktype)); - PgStat_BktypeIO *bktype_stats = &backends_io_stats->stats[bktype]; + Datum bktype_desc; + PgStat_BktypeIO *bktype_stats; + + bktype_desc = CStringGetTextDatum(GetBackendTypeDesc(bktype)); + bktype_stats = &backends_io_stats->stats[bktype]; /* * In Assert builds, we can afford an extra loop through all of the @@ -1462,6 +1465,111 @@ pg_stat_get_io(PG_FUNCTION_ARGS) return (Datum) 0; } +Datum +pg_stat_get_my_io(PG_FUNCTION_ARGS) +{ + ReturnSetInfo *rsinfo; + PgStat_Backend_IO *backend_io_stats; + Datum bktype_desc; + PgStat_BktypeIO *bktype_stats; + BackendType bktype; + Datum reset_time; + + InitMaterializedSRF(fcinfo, 0); + rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; + + backend_io_stats = pgstat_fetch_my_stat_io(); + + bktype = backend_io_stats->bktype; + bktype_desc = CStringGetTextDatum(GetBackendTypeDesc(bktype)); + bktype_stats = &backend_io_stats->stats; + reset_time = TimestampTzGetDatum(backend_io_stats->stat_reset_timestamp); + + /* + * In Assert builds, we can afford an extra loop through all of the + * counters checking that only expected stats are non-zero, since it keeps + * the non-Assert code cleaner. + */ + Assert(pgstat_bktype_io_stats_valid(bktype_stats, bktype)); + + for (int io_obj = 0; io_obj < IOOBJECT_NUM_TYPES; io_obj++) + { + const char *obj_name = pgstat_get_io_object_name(io_obj); + + for (int io_context = 0; io_context < IOCONTEXT_NUM_TYPES; io_context++) + { + const char *context_name = pgstat_get_io_context_name(io_context); + + Datum values[IO_NUM_COLUMNS] = {0}; + bool nulls[IO_NUM_COLUMNS] = {0}; + + /* + * Some combinations of BackendType, IOObject, and IOContext are + * not valid for any type of IOOp. In such cases, omit the entire + * row from the view. + */ + if (!pgstat_tracks_io_object(bktype, io_obj, io_context)) + continue; + + values[IO_COL_BACKEND_TYPE] = bktype_desc; + values[IO_COL_CONTEXT] = CStringGetTextDatum(context_name); + values[IO_COL_OBJECT] = CStringGetTextDatum(obj_name); + if (backend_io_stats->stat_reset_timestamp != 0) + values[IO_COL_RESET_TIME] = reset_time; + else + nulls[IO_COL_RESET_TIME] = true; + + /* + * Hard-code this to the value of BLCKSZ for now. Future values + * could include XLOG_BLCKSZ, once WAL IO is tracked, and constant + * multipliers, once non-block-oriented IO (e.g. temporary file + * IO) is tracked. + */ + values[IO_COL_CONVERSION] = Int64GetDatum(BLCKSZ); + + for (int io_op = 0; io_op < IOOP_NUM_TYPES; io_op++) + { + int op_idx = pgstat_get_io_op_index(io_op); + int time_idx = pgstat_get_io_time_index(io_op); + + /* + * Some combinations of BackendType and IOOp, of IOContext and + * IOOp, and of IOObject and IOOp are not tracked. Set these + * cells in the view NULL. + */ + if (pgstat_tracks_io_op(bktype, io_obj, io_context, io_op)) + { + PgStat_Counter count = + bktype_stats->counts[io_obj][io_context][io_op]; + + values[op_idx] = Int64GetDatum(count); + } + else + nulls[op_idx] = true; + + /* not every operation is timed */ + if (time_idx == IO_COL_INVALID) + continue; + + if (!nulls[op_idx]) + { + PgStat_Counter time = + bktype_stats->times[io_obj][io_context][io_op]; + + values[time_idx] = Float8GetDatum(pg_stat_us_to_ms(time)); + } + else + nulls[time_idx] = true; + } + + tuplestore_putvalues(rsinfo->setResult, rsinfo->setDesc, + values, nulls); + } + } + + return (Datum) 0; +} + /* * Returns statistics of WAL activity */ diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index ff5436acac..62658e9a00 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -5831,6 +5831,15 @@ proargnames => '{backend_type,object,context,reads,read_time,writes,write_time,writebacks,writeback_time,extends,extend_time,op_bytes,hits,evictions,reuses,fsyncs,fsync_time,stats_reset}', prosrc => 'pg_stat_get_io' }, +{ oid => '8806', descr => 'statistics: My backend IO statistics', + proname => 'pg_stat_get_my_io', prorows => '5', proretset => 't', + provolatile => 'v', proparallel => 'r', prorettype => 'record', + proargtypes => '', + proallargtypes => '{text,text,text,int8,float8,int8,float8,int8,float8,int8,float8,int8,int8,int8,int8,int8,float8,timestamptz}', + proargmodes => '{o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o}', + proargnames => '{backend_type,object,context,reads,read_time,writes,write_time,writebacks,writeback_time,extends,extend_time,op_bytes,hits,evictions,reuses,fsyncs,fsync_time,stats_reset}', + prosrc => 'pg_stat_get_my_io' }, + { oid => '1136', descr => 'statistics: information about WAL activity', proname => 'pg_stat_get_wal', proisstrict => 'f', provolatile => 's', proparallel => 'r', prorettype => 'record', proargtypes => '', diff --git a/src/include/pgstat.h b/src/include/pgstat.h index be2c91168a..9d63b1a5b7 100644 --- a/src/include/pgstat.h +++ b/src/include/pgstat.h @@ -16,6 +16,7 @@ #include "portability/instr_time.h" #include "postmaster/pgarch.h" /* for MAX_XFN_CHARS */ #include "replication/conflict.h" +#include "storage/proc.h" #include "utils/backend_progress.h" /* for backward compatibility */ #include "utils/backend_status.h" /* for backward compatibility */ #include "utils/relcache.h" @@ -76,6 +77,8 @@ */ #define PGSTAT_KIND_EXPERIMENTAL 128 +#define NumProcStatSlots (MaxBackends + NUM_AUXILIARY_PROCS) + static inline bool pgstat_is_kind_builtin(PgStat_Kind kind) { @@ -351,6 +354,12 @@ typedef struct PgStat_IO PgStat_BktypeIO stats[BACKEND_NUM_TYPES]; } PgStat_IO; +typedef struct PgStat_Backend_IO +{ + TimestampTz stat_reset_timestamp; + BackendType bktype; + PgStat_BktypeIO stats; +} PgStat_Backend_IO; typedef struct PgStat_StatDBEntry { @@ -559,7 +568,8 @@ extern instr_time pgstat_prepare_io_time(bool track_io_guc); extern void pgstat_count_io_op_time(IOObject io_object, IOContext io_context, IOOp io_op, instr_time start_time, uint32 cnt); -extern PgStat_IO *pgstat_fetch_stat_io(void); +extern PgStat_IO *pgstat_fetch_global_stat_io(void); +extern PgStat_Backend_IO *pgstat_fetch_my_stat_io(void); extern const char *pgstat_get_io_context_name(IOContext io_context); extern const char *pgstat_get_io_object_name(IOObject io_object); @@ -568,7 +578,7 @@ extern bool pgstat_tracks_io_object(BackendType bktype, IOObject io_object, IOContext io_context); extern bool pgstat_tracks_io_op(BackendType bktype, IOObject io_object, IOContext io_context, IOOp io_op); - +extern void pgstat_reset_io_counter_internal(int index, TimestampTz ts); /* * Functions in pgstat_database.c diff --git a/src/include/utils/pgstat_internal.h b/src/include/utils/pgstat_internal.h index bba90e898d..ea4cc6a2c6 100644 --- a/src/include/utils/pgstat_internal.h +++ b/src/include/utils/pgstat_internal.h @@ -366,11 +366,12 @@ typedef struct PgStatShared_Checkpointer typedef struct PgStatShared_IO { /* - * locks[i] protects stats.stats[i]. locks[0] also protects - * stats.stat_reset_timestamp. + * locks[i] protects global_stats.stats[i]. locks[0] also protects + * global_stats.stat_reset_timestamp. */ LWLock locks[BACKEND_NUM_TYPES]; - PgStat_IO stats; + PgStat_IO global_stats; + PgStat_Backend_IO stat[FLEXIBLE_ARRAY_MEMBER]; } PgStatShared_IO; typedef struct PgStatShared_SLRU @@ -463,7 +464,6 @@ typedef struct PgStat_ShmemControl PgStatShared_Archiver archiver; PgStatShared_BgWriter bgwriter; PgStatShared_Checkpointer checkpointer; - PgStatShared_IO io; PgStatShared_SLRU slru; PgStatShared_Wal wal; @@ -473,6 +473,8 @@ typedef struct PgStat_ShmemControl */ void *custom_data[PGSTAT_KIND_CUSTOM_SIZE]; + /* has to be at the end due to FLEXIBLE_ARRAY_MEMBER */ + PgStatShared_IO io; } PgStat_ShmemControl; @@ -494,7 +496,8 @@ typedef struct PgStat_Snapshot PgStat_CheckpointerStats checkpointer; - PgStat_IO io; + PgStat_Backend_IO my_io; + PgStat_IO global_io; PgStat_SLRUStats slru[SLRU_NUM_ELEMENTS]; @@ -629,6 +632,7 @@ extern bool pgstat_io_flush_cb(bool nowait); extern void pgstat_io_init_shmem_cb(void *stats); extern void pgstat_io_reset_all_cb(TimestampTz ts); extern void pgstat_io_snapshot_cb(void); +extern void pgstat_io_init_backend_cb(void); /* diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out index a1626f3fae..785298d7b0 100644 --- a/src/test/regress/expected/rules.out +++ b/src/test/regress/expected/rules.out @@ -1398,6 +1398,25 @@ pg_matviews| SELECT n.nspname AS schemaname, LEFT JOIN pg_namespace n ON ((n.oid = c.relnamespace))) LEFT JOIN pg_tablespace t ON ((t.oid = c.reltablespace))) WHERE (c.relkind = 'm'::"char"); +pg_my_stat_io| SELECT backend_type, + object, + context, + reads, + read_time, + writes, + write_time, + writebacks, + writeback_time, + extends, + extend_time, + op_bytes, + hits, + evictions, + reuses, + fsyncs, + fsync_time, + stats_reset + FROM pg_stat_get_my_io() b(backend_type, object, context, reads, read_time, writes, write_time, writebacks, writeback_time, extends, extend_time, op_bytes, hits, evictions, reuses, fsyncs, fsync_time, stats_reset); pg_policies| SELECT n.nspname AS schemaname, c.relname AS tablename, pol.polname AS policyname, diff --git a/src/test/regress/expected/stats.out b/src/test/regress/expected/stats.out index 6e08898b18..c489e528e0 100644 --- a/src/test/regress/expected/stats.out +++ b/src/test/regress/expected/stats.out @@ -1249,7 +1249,7 @@ SELECT pg_stat_get_subscription_stats(NULL); (1 row) --- Test that the following operations are tracked in pg_stat_io: +-- Test that the following operations are tracked in pg_[my]_stat_io: -- - reads of target blocks into shared buffers -- - writes of shared buffers to permanent storage -- - extends of relations using shared buffers @@ -1261,9 +1261,14 @@ SELECT pg_stat_get_subscription_stats(NULL); -- extends. SELECT sum(extends) AS io_sum_shared_before_extends FROM pg_stat_io WHERE context = 'normal' AND object = 'relation' \gset +SELECT sum(extends) AS my_io_sum_shared_before_extends + FROM pg_my_stat_io WHERE context = 'normal' AND object = 'relation' \gset SELECT sum(writes) AS writes, sum(fsyncs) AS fsyncs FROM pg_stat_io WHERE object = 'relation' \gset io_sum_shared_before_ +SELECT sum(writes) AS writes, sum(fsyncs) AS fsyncs + FROM pg_my_stat_io + WHERE object = 'relation' \gset my_io_sum_shared_before_ CREATE TABLE test_io_shared(a int); INSERT INTO test_io_shared SELECT i FROM generate_series(1,100)i; SELECT pg_stat_force_next_flush(); @@ -1280,8 +1285,16 @@ SELECT :io_sum_shared_after_extends > :io_sum_shared_before_extends; t (1 row) +SELECT sum(extends) AS my_io_sum_shared_after_extends + FROM pg_my_stat_io WHERE context = 'normal' AND object = 'relation' \gset +SELECT :my_io_sum_shared_after_extends > :my_io_sum_shared_before_extends; + ?column? +---------- + t +(1 row) + -- After a checkpoint, there should be some additional IOCONTEXT_NORMAL writes --- and fsyncs. +-- and fsyncs in the global stats (not for the backend). -- See comment above for rationale for two explicit CHECKPOINTs. CHECKPOINT; CHECKPOINT; @@ -1301,6 +1314,23 @@ SELECT current_setting('fsync') = 'off' t (1 row) +SELECT sum(writes) AS writes, sum(fsyncs) AS fsyncs + FROM pg_my_stat_io + WHERE object = 'relation' \gset my_io_sum_shared_after_ +SELECT :my_io_sum_shared_after_writes >= :my_io_sum_shared_before_writes; + ?column? +---------- + t +(1 row) + +SELECT current_setting('fsync') = 'off' + OR (:my_io_sum_shared_after_fsyncs = :my_io_sum_shared_before_fsyncs + AND :my_io_sum_shared_after_fsyncs= 0); + ?column? +---------- + t +(1 row) + -- Change the tablespace so that the table is rewritten directly, then SELECT -- from it to cause it to be read back into shared buffers. SELECT sum(reads) AS io_sum_shared_before_reads @@ -1521,6 +1551,8 @@ SELECT pg_stat_have_stats('io', 0, 0); SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS io_stats_pre_reset FROM pg_stat_io \gset +SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS my_io_stats_pre_reset + FROM pg_my_stat_io \gset SELECT pg_stat_reset_shared('io'); pg_stat_reset_shared ---------------------- @@ -1535,6 +1567,14 @@ SELECT :io_stats_post_reset < :io_stats_pre_reset; t (1 row) +SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS my_io_stats_post_reset + FROM pg_my_stat_io \gset +SELECT :my_io_stats_post_reset < :my_io_stats_pre_reset; + ?column? +---------- + t +(1 row) + -- test BRIN index doesn't block HOT update CREATE TABLE brin_hot ( id integer PRIMARY KEY, diff --git a/src/test/regress/sql/stats.sql b/src/test/regress/sql/stats.sql index d8ac0d06f4..c95cb71652 100644 --- a/src/test/regress/sql/stats.sql +++ b/src/test/regress/sql/stats.sql @@ -595,7 +595,7 @@ SELECT pg_stat_get_replication_slot(NULL); SELECT pg_stat_get_subscription_stats(NULL); --- Test that the following operations are tracked in pg_stat_io: +-- Test that the following operations are tracked in pg_[my]_stat_io: -- - reads of target blocks into shared buffers -- - writes of shared buffers to permanent storage -- - extends of relations using shared buffers @@ -609,18 +609,26 @@ SELECT pg_stat_get_subscription_stats(NULL); -- extends. SELECT sum(extends) AS io_sum_shared_before_extends FROM pg_stat_io WHERE context = 'normal' AND object = 'relation' \gset +SELECT sum(extends) AS my_io_sum_shared_before_extends + FROM pg_my_stat_io WHERE context = 'normal' AND object = 'relation' \gset SELECT sum(writes) AS writes, sum(fsyncs) AS fsyncs FROM pg_stat_io WHERE object = 'relation' \gset io_sum_shared_before_ +SELECT sum(writes) AS writes, sum(fsyncs) AS fsyncs + FROM pg_my_stat_io + WHERE object = 'relation' \gset my_io_sum_shared_before_ CREATE TABLE test_io_shared(a int); INSERT INTO test_io_shared SELECT i FROM generate_series(1,100)i; SELECT pg_stat_force_next_flush(); SELECT sum(extends) AS io_sum_shared_after_extends FROM pg_stat_io WHERE context = 'normal' AND object = 'relation' \gset SELECT :io_sum_shared_after_extends > :io_sum_shared_before_extends; +SELECT sum(extends) AS my_io_sum_shared_after_extends + FROM pg_my_stat_io WHERE context = 'normal' AND object = 'relation' \gset +SELECT :my_io_sum_shared_after_extends > :my_io_sum_shared_before_extends; -- After a checkpoint, there should be some additional IOCONTEXT_NORMAL writes --- and fsyncs. +-- and fsyncs in the global stats (not for the backend). -- See comment above for rationale for two explicit CHECKPOINTs. CHECKPOINT; CHECKPOINT; @@ -630,7 +638,13 @@ SELECT sum(writes) AS writes, sum(fsyncs) AS fsyncs SELECT :io_sum_shared_after_writes > :io_sum_shared_before_writes; SELECT current_setting('fsync') = 'off' OR :io_sum_shared_after_fsyncs > :io_sum_shared_before_fsyncs; - +SELECT sum(writes) AS writes, sum(fsyncs) AS fsyncs + FROM pg_my_stat_io + WHERE object = 'relation' \gset my_io_sum_shared_after_ +SELECT :my_io_sum_shared_after_writes >= :my_io_sum_shared_before_writes; +SELECT current_setting('fsync') = 'off' + OR (:my_io_sum_shared_after_fsyncs = :my_io_sum_shared_before_fsyncs + AND :my_io_sum_shared_after_fsyncs= 0); -- Change the tablespace so that the table is rewritten directly, then SELECT -- from it to cause it to be read back into shared buffers. SELECT sum(reads) AS io_sum_shared_before_reads @@ -762,10 +776,15 @@ SELECT :io_sum_bulkwrite_strategy_extends_after > :io_sum_bulkwrite_strategy_ext SELECT pg_stat_have_stats('io', 0, 0); SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS io_stats_pre_reset FROM pg_stat_io \gset +SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS my_io_stats_pre_reset + FROM pg_my_stat_io \gset SELECT pg_stat_reset_shared('io'); SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS io_stats_post_reset FROM pg_stat_io \gset SELECT :io_stats_post_reset < :io_stats_pre_reset; +SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS my_io_stats_post_reset + FROM pg_my_stat_io \gset +SELECT :my_io_stats_post_reset < :my_io_stats_pre_reset; -- test BRIN index doesn't block HOT update -- 2.34.1 --zQrxP4xQumq+YTTN Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v3-0002-Add-pg_stat_get_backend_io.patch" ^ permalink raw reply [nested|flat] 25+ messages in thread
* [PATCH v4 1/2] per backend I/O statistics @ 2024-08-22 15:16 Bertrand Drouvot <[email protected]> 0 siblings, 0 replies; 25+ messages in thread From: Bertrand Drouvot @ 2024-08-22 15:16 UTC (permalink / raw) While pg_stat_io provides cluster-wide I/O statistics, this commit adds a new pg_my_stat_io view to display "my" backend I/O statistics. The KIND_IO stats are still "fixed amount" ones as the maximum number of backend is fixed. The statistics snapshot is made for the global stats (the aggregated ones) and for my backend stats. The snapshot is not build in each backend to copy all the others backends stats, as 1/ there is no use case (there is no need to get others backends I/O statistics while taking care of the stats_fetch_consistency) and 2/ that could be memory expensive depending of the number of max connections. A subsequent commit will add a new pg_stat_get_backend_io() function to be able to retrieve the I/O statistics for a given backend pid (each execution re-fetches counters from shared memory because as stated above there is no snapshots being created in each backend to copy the other backends stats). XXX: Bump catalog version needs to be done. --- doc/src/sgml/config.sgml | 4 +- doc/src/sgml/monitoring.sgml | 28 ++++++ src/backend/catalog/system_views.sql | 22 +++++ src/backend/utils/activity/pgstat.c | 11 ++- src/backend/utils/activity/pgstat_io.c | 93 +++++++++++++++--- src/backend/utils/activity/pgstat_shmem.c | 4 +- src/backend/utils/adt/pgstatfuncs.c | 114 +++++++++++++++++++++- src/include/catalog/pg_proc.dat | 9 ++ src/include/pgstat.h | 14 ++- src/include/utils/pgstat_internal.h | 14 ++- src/test/regress/expected/rules.out | 19 ++++ src/test/regress/expected/stats.out | 44 ++++++++- src/test/regress/sql/stats.sql | 25 ++++- 13 files changed, 368 insertions(+), 33 deletions(-) 8.4% doc/src/sgml/ 29.9% src/backend/utils/activity/ 23.7% src/backend/utils/adt/ 4.4% src/include/catalog/ 3.1% src/include/utils/ 3.1% src/include/ 14.3% src/test/regress/expected/ 9.9% src/test/regress/sql/ diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 0aec11f443..2a59b97093 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -8331,7 +8331,9 @@ COPY postgres_log FROM '/full/path/to/logfile.csv' WITH csv; displayed in <link linkend="monitoring-pg-stat-database-view"> <structname>pg_stat_database</structname></link>, <link linkend="monitoring-pg-stat-io-view"> - <structname>pg_stat_io</structname></link>, in the output of + <structname>pg_stat_io</structname></link>, + <link linkend="monitoring-pg-my-stat-io-view"> + <structname>pg_my_stat_io</structname></link>, in the output of <xref linkend="sql-explain"/> when the <literal>BUFFERS</literal> option is used, in the output of <xref linkend="sql-vacuum"/> when the <literal>VERBOSE</literal> option is used, by autovacuum diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml index 933de6fe07..b28ca4e0ca 100644 --- a/doc/src/sgml/monitoring.sgml +++ b/doc/src/sgml/monitoring.sgml @@ -488,6 +488,16 @@ postgres 27093 0.0 0.0 30096 2752 ? Ss 11:34 0:00 postgres: ser </entry> </row> + <row> + <entry><structname>pg_my_stat_io</structname><indexterm><primary>pg_my_stat_io</primary></indexterm></entry> + <entry> + One row for each combination of context and target object containing + my backend I/O statistics. + See <link linkend="monitoring-pg-my-stat-io-view"> + <structname>pg_my_stat_io</structname></link> for details. + </entry> + </row> + <row> <entry><structname>pg_stat_replication_slots</structname><indexterm><primary>pg_stat_replication_slots</primary></indexterm></entry> <entry>One row per replication slot, showing statistics about the @@ -2948,6 +2958,24 @@ description | Waiting for a newly initialized WAL file to reach durable storage + </sect2> + + <sect2 id="monitoring-pg-my-stat-io-view"> + <title><structname>pg_my_stat_io</structname></title> + + <indexterm> + <primary>pg_my_stat_io</primary> + </indexterm> + + <para> + The <structname>pg_my_stat_io</structname> view will contain one row for each + combination of target I/O object and I/O context, showing + my backend I/O statistics. Combinations which do not make sense are + omitted. The fields are exactly the same as the ones in the <link + linkend="monitoring-pg-stat-io-view"> <structname>pg_stat_io</structname></link> + view. + </para> + </sect2> <sect2 id="monitoring-pg-stat-bgwriter-view"> diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index 7fd5d256a1..b4939b7bc6 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1168,6 +1168,28 @@ SELECT b.stats_reset FROM pg_stat_get_io() b; +CREATE VIEW pg_my_stat_io AS +SELECT + b.backend_type, + b.object, + b.context, + b.reads, + b.read_time, + b.writes, + b.write_time, + b.writebacks, + b.writeback_time, + b.extends, + b.extend_time, + b.op_bytes, + b.hits, + b.evictions, + b.reuses, + b.fsyncs, + b.fsync_time, + b.stats_reset +FROM pg_stat_get_my_io() b; + CREATE VIEW pg_stat_wal AS SELECT w.wal_records, diff --git a/src/backend/utils/activity/pgstat.c b/src/backend/utils/activity/pgstat.c index a7f2dfc744..fda4ca9a4a 100644 --- a/src/backend/utils/activity/pgstat.c +++ b/src/backend/utils/activity/pgstat.c @@ -406,10 +406,12 @@ static const PgStat_KindInfo pgstat_kind_builtin_infos[PGSTAT_KIND_BUILTIN_SIZE] .fixed_amount = true, - .snapshot_ctl_off = offsetof(PgStat_Snapshot, io), + .init_backend_cb = pgstat_io_init_backend_cb, + .snapshot_ctl_off = offsetof(PgStat_Snapshot, global_io), .shared_ctl_off = offsetof(PgStat_ShmemControl, io), - .shared_data_off = offsetof(PgStatShared_IO, stats), - .shared_data_len = sizeof(((PgStatShared_IO *) 0)->stats), + /* [de-]serialize global_stats only */ + .shared_data_off = offsetof(PgStatShared_IO, global_stats), + .shared_data_len = sizeof(((PgStatShared_IO *) 0)->global_stats), .flush_fixed_cb = pgstat_io_flush_cb, .have_fixed_pending_cb = pgstat_io_have_pending_cb, @@ -587,6 +589,9 @@ pgstat_shutdown_hook(int code, Datum arg) pgstat_report_stat(true); + /* reset the pgstat_io counter related to this proc number */ + pgstat_reset_io_counter_internal(MyProcNumber, 0); + /* there shouldn't be any pending changes left */ Assert(dlist_is_empty(&pgStatPending)); dlist_init(&pgStatPending); diff --git a/src/backend/utils/activity/pgstat_io.c b/src/backend/utils/activity/pgstat_io.c index cc2ffc78aa..d32121fc1f 100644 --- a/src/backend/utils/activity/pgstat_io.c +++ b/src/backend/utils/activity/pgstat_io.c @@ -31,6 +31,7 @@ typedef struct PgStat_PendingIO static PgStat_PendingIO PendingIOStats; static bool have_iostats = false; +void pgstat_reset_io_counter_internal(int index, TimestampTz ts); /* * Check that stats have not been counted for any combination of IOObject, @@ -154,11 +155,19 @@ pgstat_count_io_op_time(IOObject io_object, IOContext io_context, IOOp io_op, } PgStat_IO * -pgstat_fetch_stat_io(void) +pgstat_fetch_global_stat_io(void) { pgstat_snapshot_fixed(PGSTAT_KIND_IO); - return &pgStatLocal.snapshot.io; + return &pgStatLocal.snapshot.global_io; +} + +PgStat_Backend_IO * +pgstat_fetch_my_stat_io(void) +{ + pgstat_snapshot_fixed(PGSTAT_KIND_IO); + + return &pgStatLocal.snapshot.my_io; } /* @@ -192,13 +201,16 @@ pgstat_io_flush_cb(bool nowait) { LWLock *bktype_lock; PgStat_BktypeIO *bktype_shstats; + PgStat_BktypeIO *global_bktype_shstats; if (!have_iostats) return false; bktype_lock = &pgStatLocal.shmem->io.locks[MyBackendType]; bktype_shstats = - &pgStatLocal.shmem->io.stats.stats[MyBackendType]; + &pgStatLocal.shmem->io.stat[MyProcNumber].stats; + global_bktype_shstats = + &pgStatLocal.shmem->io.global_stats.stats[MyBackendType]; if (!nowait) LWLockAcquire(bktype_lock, LW_EXCLUSIVE); @@ -212,19 +224,28 @@ pgstat_io_flush_cb(bool nowait) for (int io_op = 0; io_op < IOOP_NUM_TYPES; io_op++) { instr_time time; + PgStat_Counter counter; + + counter = PendingIOStats.counts[io_object][io_context][io_op]; - bktype_shstats->counts[io_object][io_context][io_op] += - PendingIOStats.counts[io_object][io_context][io_op]; + bktype_shstats->counts[io_object][io_context][io_op] += counter; + + global_bktype_shstats->counts[io_object][io_context][io_op] += + counter; time = PendingIOStats.pending_times[io_object][io_context][io_op]; bktype_shstats->times[io_object][io_context][io_op] += INSTR_TIME_GET_MICROSEC(time); + + global_bktype_shstats->times[io_object][io_context][io_op] += + INSTR_TIME_GET_MICROSEC(time); } } } Assert(pgstat_bktype_io_stats_valid(bktype_shstats, MyBackendType)); + Assert(pgstat_bktype_io_stats_valid(global_bktype_shstats, MyBackendType)); LWLockRelease(bktype_lock); @@ -278,13 +299,35 @@ pgstat_io_init_shmem_cb(void *stats) LWLockInitialize(&stat_shmem->locks[i], LWTRANCHE_PGSTATS_DATA); } +void +pgstat_reset_io_counter_internal(int index, TimestampTz ts) +{ + LWLock *bktype_lock = &pgStatLocal.shmem->io.locks[0]; + PgStat_BktypeIO *bktype_shstats = &pgStatLocal.shmem->io.stat[index].stats; + + + /* + * Use the lock in the first BackendType's PgStat_BktypeIO to protect the + * reset timestamp as well. + */ + LWLockAcquire(bktype_lock, LW_EXCLUSIVE); + + pgStatLocal.shmem->io.stat[index].stat_reset_timestamp = ts; + + memset(bktype_shstats, 0, sizeof(*bktype_shstats)); + LWLockRelease(bktype_lock); +} + void pgstat_io_reset_all_cb(TimestampTz ts) { + for (int i = 0; i < NumProcStatSlots; i++) + pgstat_reset_io_counter_internal(i, ts); + for (int i = 0; i < BACKEND_NUM_TYPES; i++) { LWLock *bktype_lock = &pgStatLocal.shmem->io.locks[i]; - PgStat_BktypeIO *bktype_shstats = &pgStatLocal.shmem->io.stats.stats[i]; + PgStat_BktypeIO *bktype_shstats = &pgStatLocal.shmem->io.global_stats.stats[i]; LWLockAcquire(bktype_lock, LW_EXCLUSIVE); @@ -293,7 +336,7 @@ pgstat_io_reset_all_cb(TimestampTz ts) * the reset timestamp as well. */ if (i == 0) - pgStatLocal.shmem->io.stats.stat_reset_timestamp = ts; + pgStatLocal.shmem->io.global_stats.stat_reset_timestamp = ts; memset(bktype_shstats, 0, sizeof(*bktype_shstats)); LWLockRelease(bktype_lock); @@ -303,11 +346,28 @@ pgstat_io_reset_all_cb(TimestampTz ts) void pgstat_io_snapshot_cb(void) { + /* First, our own stats */ + PgStat_BktypeIO *bktype_shstats = &pgStatLocal.shmem->io.stat[MyProcNumber].stats; + PgStat_BktypeIO *bktype_snap = &pgStatLocal.snapshot.my_io.stats; + LWLock *mybktype_lock = &pgStatLocal.shmem->io.locks[0]; + + LWLockAcquire(mybktype_lock, LW_SHARED); + + pgStatLocal.snapshot.my_io.stat_reset_timestamp = + pgStatLocal.shmem->io.stat[MyProcNumber].stat_reset_timestamp; + + pgStatLocal.snapshot.my_io.bktype = + pgStatLocal.shmem->io.stat[MyProcNumber].bktype; + + *bktype_snap = *bktype_shstats; + LWLockRelease(mybktype_lock); + + /* Now, the global stats */ for (int i = 0; i < BACKEND_NUM_TYPES; i++) { LWLock *bktype_lock = &pgStatLocal.shmem->io.locks[i]; - PgStat_BktypeIO *bktype_shstats = &pgStatLocal.shmem->io.stats.stats[i]; - PgStat_BktypeIO *bktype_snap = &pgStatLocal.snapshot.io.stats[i]; + PgStat_BktypeIO *bktype_global_shstats = &pgStatLocal.shmem->io.global_stats.stats[i]; + PgStat_BktypeIO *bktype_global_snap = &pgStatLocal.snapshot.global_io.stats[i]; LWLockAcquire(bktype_lock, LW_SHARED); @@ -316,11 +376,11 @@ pgstat_io_snapshot_cb(void) * the reset timestamp as well. */ if (i == 0) - pgStatLocal.snapshot.io.stat_reset_timestamp = - pgStatLocal.shmem->io.stats.stat_reset_timestamp; + pgStatLocal.snapshot.global_io.stat_reset_timestamp = + pgStatLocal.shmem->io.global_stats.stat_reset_timestamp; /* using struct assignment due to better type safety */ - *bktype_snap = *bktype_shstats; + *bktype_global_snap = *bktype_global_shstats; LWLockRelease(bktype_lock); } } @@ -503,3 +563,12 @@ pgstat_tracks_io_op(BackendType bktype, IOObject io_object, return true; } + +void +pgstat_io_init_backend_cb(void) +{ + /* Initialize our backend type in the IO statistics */ + pgStatLocal.shmem->io.stat[MyProcNumber].bktype = MyBackendType; + /* Set the stat_reset_timestamp to zero */ + pgStatLocal.shmem->io.stat[MyProcNumber].stat_reset_timestamp = 0; +} diff --git a/src/backend/utils/activity/pgstat_shmem.c b/src/backend/utils/activity/pgstat_shmem.c index ec93bf6902..3cfcba9609 100644 --- a/src/backend/utils/activity/pgstat_shmem.c +++ b/src/backend/utils/activity/pgstat_shmem.c @@ -128,7 +128,7 @@ StatsShmemSize(void) { Size sz; - sz = MAXALIGN(sizeof(PgStat_ShmemControl)); + sz = MAXALIGN(sizeof(PgStat_ShmemControl) + mul_size(sizeof(PgStat_Backend_IO), NumProcStatSlots)); sz = add_size(sz, pgstat_dsa_init_size()); /* Add shared memory for all the custom fixed-numbered statistics */ @@ -172,7 +172,7 @@ StatsShmemInit(void) Assert(!found); /* the allocation of pgStatLocal.shmem itself */ - p += MAXALIGN(sizeof(PgStat_ShmemControl)); + p += MAXALIGN(sizeof(PgStat_ShmemControl) + mul_size(sizeof(PgStat_Backend_IO), NumProcStatSlots)); /* * Create a small dsa allocation in plain shared memory. This is diff --git a/src/backend/utils/adt/pgstatfuncs.c b/src/backend/utils/adt/pgstatfuncs.c index 33c7b25560..4c00570396 100644 --- a/src/backend/utils/adt/pgstatfuncs.c +++ b/src/backend/utils/adt/pgstatfuncs.c @@ -1363,14 +1363,17 @@ pg_stat_get_io(PG_FUNCTION_ARGS) InitMaterializedSRF(fcinfo, 0); rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; - backends_io_stats = pgstat_fetch_stat_io(); + backends_io_stats = pgstat_fetch_global_stat_io(); reset_time = TimestampTzGetDatum(backends_io_stats->stat_reset_timestamp); for (int bktype = 0; bktype < BACKEND_NUM_TYPES; bktype++) { - Datum bktype_desc = CStringGetTextDatum(GetBackendTypeDesc(bktype)); - PgStat_BktypeIO *bktype_stats = &backends_io_stats->stats[bktype]; + Datum bktype_desc; + PgStat_BktypeIO *bktype_stats; + + bktype_desc = CStringGetTextDatum(GetBackendTypeDesc(bktype)); + bktype_stats = &backends_io_stats->stats[bktype]; /* * In Assert builds, we can afford an extra loop through all of the @@ -1462,6 +1465,111 @@ pg_stat_get_io(PG_FUNCTION_ARGS) return (Datum) 0; } +Datum +pg_stat_get_my_io(PG_FUNCTION_ARGS) +{ + ReturnSetInfo *rsinfo; + PgStat_Backend_IO *backend_io_stats; + Datum bktype_desc; + PgStat_BktypeIO *bktype_stats; + BackendType bktype; + Datum reset_time; + + InitMaterializedSRF(fcinfo, 0); + rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; + + backend_io_stats = pgstat_fetch_my_stat_io(); + + bktype = backend_io_stats->bktype; + bktype_desc = CStringGetTextDatum(GetBackendTypeDesc(bktype)); + bktype_stats = &backend_io_stats->stats; + reset_time = TimestampTzGetDatum(backend_io_stats->stat_reset_timestamp); + + /* + * In Assert builds, we can afford an extra loop through all of the + * counters checking that only expected stats are non-zero, since it keeps + * the non-Assert code cleaner. + */ + Assert(pgstat_bktype_io_stats_valid(bktype_stats, bktype)); + + for (int io_obj = 0; io_obj < IOOBJECT_NUM_TYPES; io_obj++) + { + const char *obj_name = pgstat_get_io_object_name(io_obj); + + for (int io_context = 0; io_context < IOCONTEXT_NUM_TYPES; io_context++) + { + const char *context_name = pgstat_get_io_context_name(io_context); + + Datum values[IO_NUM_COLUMNS] = {0}; + bool nulls[IO_NUM_COLUMNS] = {0}; + + /* + * Some combinations of BackendType, IOObject, and IOContext are + * not valid for any type of IOOp. In such cases, omit the entire + * row from the view. + */ + if (!pgstat_tracks_io_object(bktype, io_obj, io_context)) + continue; + + values[IO_COL_BACKEND_TYPE] = bktype_desc; + values[IO_COL_CONTEXT] = CStringGetTextDatum(context_name); + values[IO_COL_OBJECT] = CStringGetTextDatum(obj_name); + if (backend_io_stats->stat_reset_timestamp != 0) + values[IO_COL_RESET_TIME] = reset_time; + else + nulls[IO_COL_RESET_TIME] = true; + + /* + * Hard-code this to the value of BLCKSZ for now. Future values + * could include XLOG_BLCKSZ, once WAL IO is tracked, and constant + * multipliers, once non-block-oriented IO (e.g. temporary file + * IO) is tracked. + */ + values[IO_COL_CONVERSION] = Int64GetDatum(BLCKSZ); + + for (int io_op = 0; io_op < IOOP_NUM_TYPES; io_op++) + { + int op_idx = pgstat_get_io_op_index(io_op); + int time_idx = pgstat_get_io_time_index(io_op); + + /* + * Some combinations of BackendType and IOOp, of IOContext and + * IOOp, and of IOObject and IOOp are not tracked. Set these + * cells in the view NULL. + */ + if (pgstat_tracks_io_op(bktype, io_obj, io_context, io_op)) + { + PgStat_Counter count = + bktype_stats->counts[io_obj][io_context][io_op]; + + values[op_idx] = Int64GetDatum(count); + } + else + nulls[op_idx] = true; + + /* not every operation is timed */ + if (time_idx == IO_COL_INVALID) + continue; + + if (!nulls[op_idx]) + { + PgStat_Counter time = + bktype_stats->times[io_obj][io_context][io_op]; + + values[time_idx] = Float8GetDatum(pg_stat_us_to_ms(time)); + } + else + nulls[time_idx] = true; + } + + tuplestore_putvalues(rsinfo->setResult, rsinfo->setDesc, + values, nulls); + } + } + + return (Datum) 0; +} + /* * Returns statistics of WAL activity */ diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index 53a081ed88..4d5af3d7cf 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -5880,6 +5880,15 @@ proargnames => '{backend_type,object,context,reads,read_time,writes,write_time,writebacks,writeback_time,extends,extend_time,op_bytes,hits,evictions,reuses,fsyncs,fsync_time,stats_reset}', prosrc => 'pg_stat_get_io' }, +{ oid => '8806', descr => 'statistics: My backend IO statistics', + proname => 'pg_stat_get_my_io', prorows => '5', proretset => 't', + provolatile => 'v', proparallel => 'r', prorettype => 'record', + proargtypes => '', + proallargtypes => '{text,text,text,int8,float8,int8,float8,int8,float8,int8,float8,int8,int8,int8,int8,int8,float8,timestamptz}', + proargmodes => '{o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o}', + proargnames => '{backend_type,object,context,reads,read_time,writes,write_time,writebacks,writeback_time,extends,extend_time,op_bytes,hits,evictions,reuses,fsyncs,fsync_time,stats_reset}', + prosrc => 'pg_stat_get_my_io' }, + { oid => '1136', descr => 'statistics: information about WAL activity', proname => 'pg_stat_get_wal', proisstrict => 'f', provolatile => 's', proparallel => 'r', prorettype => 'record', proargtypes => '', diff --git a/src/include/pgstat.h b/src/include/pgstat.h index be2c91168a..9d63b1a5b7 100644 --- a/src/include/pgstat.h +++ b/src/include/pgstat.h @@ -16,6 +16,7 @@ #include "portability/instr_time.h" #include "postmaster/pgarch.h" /* for MAX_XFN_CHARS */ #include "replication/conflict.h" +#include "storage/proc.h" #include "utils/backend_progress.h" /* for backward compatibility */ #include "utils/backend_status.h" /* for backward compatibility */ #include "utils/relcache.h" @@ -76,6 +77,8 @@ */ #define PGSTAT_KIND_EXPERIMENTAL 128 +#define NumProcStatSlots (MaxBackends + NUM_AUXILIARY_PROCS) + static inline bool pgstat_is_kind_builtin(PgStat_Kind kind) { @@ -351,6 +354,12 @@ typedef struct PgStat_IO PgStat_BktypeIO stats[BACKEND_NUM_TYPES]; } PgStat_IO; +typedef struct PgStat_Backend_IO +{ + TimestampTz stat_reset_timestamp; + BackendType bktype; + PgStat_BktypeIO stats; +} PgStat_Backend_IO; typedef struct PgStat_StatDBEntry { @@ -559,7 +568,8 @@ extern instr_time pgstat_prepare_io_time(bool track_io_guc); extern void pgstat_count_io_op_time(IOObject io_object, IOContext io_context, IOOp io_op, instr_time start_time, uint32 cnt); -extern PgStat_IO *pgstat_fetch_stat_io(void); +extern PgStat_IO *pgstat_fetch_global_stat_io(void); +extern PgStat_Backend_IO *pgstat_fetch_my_stat_io(void); extern const char *pgstat_get_io_context_name(IOContext io_context); extern const char *pgstat_get_io_object_name(IOObject io_object); @@ -568,7 +578,7 @@ extern bool pgstat_tracks_io_object(BackendType bktype, IOObject io_object, IOContext io_context); extern bool pgstat_tracks_io_op(BackendType bktype, IOObject io_object, IOContext io_context, IOOp io_op); - +extern void pgstat_reset_io_counter_internal(int index, TimestampTz ts); /* * Functions in pgstat_database.c diff --git a/src/include/utils/pgstat_internal.h b/src/include/utils/pgstat_internal.h index bba90e898d..ea4cc6a2c6 100644 --- a/src/include/utils/pgstat_internal.h +++ b/src/include/utils/pgstat_internal.h @@ -366,11 +366,12 @@ typedef struct PgStatShared_Checkpointer typedef struct PgStatShared_IO { /* - * locks[i] protects stats.stats[i]. locks[0] also protects - * stats.stat_reset_timestamp. + * locks[i] protects global_stats.stats[i]. locks[0] also protects + * global_stats.stat_reset_timestamp. */ LWLock locks[BACKEND_NUM_TYPES]; - PgStat_IO stats; + PgStat_IO global_stats; + PgStat_Backend_IO stat[FLEXIBLE_ARRAY_MEMBER]; } PgStatShared_IO; typedef struct PgStatShared_SLRU @@ -463,7 +464,6 @@ typedef struct PgStat_ShmemControl PgStatShared_Archiver archiver; PgStatShared_BgWriter bgwriter; PgStatShared_Checkpointer checkpointer; - PgStatShared_IO io; PgStatShared_SLRU slru; PgStatShared_Wal wal; @@ -473,6 +473,8 @@ typedef struct PgStat_ShmemControl */ void *custom_data[PGSTAT_KIND_CUSTOM_SIZE]; + /* has to be at the end due to FLEXIBLE_ARRAY_MEMBER */ + PgStatShared_IO io; } PgStat_ShmemControl; @@ -494,7 +496,8 @@ typedef struct PgStat_Snapshot PgStat_CheckpointerStats checkpointer; - PgStat_IO io; + PgStat_Backend_IO my_io; + PgStat_IO global_io; PgStat_SLRUStats slru[SLRU_NUM_ELEMENTS]; @@ -629,6 +632,7 @@ extern bool pgstat_io_flush_cb(bool nowait); extern void pgstat_io_init_shmem_cb(void *stats); extern void pgstat_io_reset_all_cb(TimestampTz ts); extern void pgstat_io_snapshot_cb(void); +extern void pgstat_io_init_backend_cb(void); /* diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out index a1626f3fae..785298d7b0 100644 --- a/src/test/regress/expected/rules.out +++ b/src/test/regress/expected/rules.out @@ -1398,6 +1398,25 @@ pg_matviews| SELECT n.nspname AS schemaname, LEFT JOIN pg_namespace n ON ((n.oid = c.relnamespace))) LEFT JOIN pg_tablespace t ON ((t.oid = c.reltablespace))) WHERE (c.relkind = 'm'::"char"); +pg_my_stat_io| SELECT backend_type, + object, + context, + reads, + read_time, + writes, + write_time, + writebacks, + writeback_time, + extends, + extend_time, + op_bytes, + hits, + evictions, + reuses, + fsyncs, + fsync_time, + stats_reset + FROM pg_stat_get_my_io() b(backend_type, object, context, reads, read_time, writes, write_time, writebacks, writeback_time, extends, extend_time, op_bytes, hits, evictions, reuses, fsyncs, fsync_time, stats_reset); pg_policies| SELECT n.nspname AS schemaname, c.relname AS tablename, pol.polname AS policyname, diff --git a/src/test/regress/expected/stats.out b/src/test/regress/expected/stats.out index 6e08898b18..c489e528e0 100644 --- a/src/test/regress/expected/stats.out +++ b/src/test/regress/expected/stats.out @@ -1249,7 +1249,7 @@ SELECT pg_stat_get_subscription_stats(NULL); (1 row) --- Test that the following operations are tracked in pg_stat_io: +-- Test that the following operations are tracked in pg_[my]_stat_io: -- - reads of target blocks into shared buffers -- - writes of shared buffers to permanent storage -- - extends of relations using shared buffers @@ -1261,9 +1261,14 @@ SELECT pg_stat_get_subscription_stats(NULL); -- extends. SELECT sum(extends) AS io_sum_shared_before_extends FROM pg_stat_io WHERE context = 'normal' AND object = 'relation' \gset +SELECT sum(extends) AS my_io_sum_shared_before_extends + FROM pg_my_stat_io WHERE context = 'normal' AND object = 'relation' \gset SELECT sum(writes) AS writes, sum(fsyncs) AS fsyncs FROM pg_stat_io WHERE object = 'relation' \gset io_sum_shared_before_ +SELECT sum(writes) AS writes, sum(fsyncs) AS fsyncs + FROM pg_my_stat_io + WHERE object = 'relation' \gset my_io_sum_shared_before_ CREATE TABLE test_io_shared(a int); INSERT INTO test_io_shared SELECT i FROM generate_series(1,100)i; SELECT pg_stat_force_next_flush(); @@ -1280,8 +1285,16 @@ SELECT :io_sum_shared_after_extends > :io_sum_shared_before_extends; t (1 row) +SELECT sum(extends) AS my_io_sum_shared_after_extends + FROM pg_my_stat_io WHERE context = 'normal' AND object = 'relation' \gset +SELECT :my_io_sum_shared_after_extends > :my_io_sum_shared_before_extends; + ?column? +---------- + t +(1 row) + -- After a checkpoint, there should be some additional IOCONTEXT_NORMAL writes --- and fsyncs. +-- and fsyncs in the global stats (not for the backend). -- See comment above for rationale for two explicit CHECKPOINTs. CHECKPOINT; CHECKPOINT; @@ -1301,6 +1314,23 @@ SELECT current_setting('fsync') = 'off' t (1 row) +SELECT sum(writes) AS writes, sum(fsyncs) AS fsyncs + FROM pg_my_stat_io + WHERE object = 'relation' \gset my_io_sum_shared_after_ +SELECT :my_io_sum_shared_after_writes >= :my_io_sum_shared_before_writes; + ?column? +---------- + t +(1 row) + +SELECT current_setting('fsync') = 'off' + OR (:my_io_sum_shared_after_fsyncs = :my_io_sum_shared_before_fsyncs + AND :my_io_sum_shared_after_fsyncs= 0); + ?column? +---------- + t +(1 row) + -- Change the tablespace so that the table is rewritten directly, then SELECT -- from it to cause it to be read back into shared buffers. SELECT sum(reads) AS io_sum_shared_before_reads @@ -1521,6 +1551,8 @@ SELECT pg_stat_have_stats('io', 0, 0); SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS io_stats_pre_reset FROM pg_stat_io \gset +SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS my_io_stats_pre_reset + FROM pg_my_stat_io \gset SELECT pg_stat_reset_shared('io'); pg_stat_reset_shared ---------------------- @@ -1535,6 +1567,14 @@ SELECT :io_stats_post_reset < :io_stats_pre_reset; t (1 row) +SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS my_io_stats_post_reset + FROM pg_my_stat_io \gset +SELECT :my_io_stats_post_reset < :my_io_stats_pre_reset; + ?column? +---------- + t +(1 row) + -- test BRIN index doesn't block HOT update CREATE TABLE brin_hot ( id integer PRIMARY KEY, diff --git a/src/test/regress/sql/stats.sql b/src/test/regress/sql/stats.sql index d8ac0d06f4..c95cb71652 100644 --- a/src/test/regress/sql/stats.sql +++ b/src/test/regress/sql/stats.sql @@ -595,7 +595,7 @@ SELECT pg_stat_get_replication_slot(NULL); SELECT pg_stat_get_subscription_stats(NULL); --- Test that the following operations are tracked in pg_stat_io: +-- Test that the following operations are tracked in pg_[my]_stat_io: -- - reads of target blocks into shared buffers -- - writes of shared buffers to permanent storage -- - extends of relations using shared buffers @@ -609,18 +609,26 @@ SELECT pg_stat_get_subscription_stats(NULL); -- extends. SELECT sum(extends) AS io_sum_shared_before_extends FROM pg_stat_io WHERE context = 'normal' AND object = 'relation' \gset +SELECT sum(extends) AS my_io_sum_shared_before_extends + FROM pg_my_stat_io WHERE context = 'normal' AND object = 'relation' \gset SELECT sum(writes) AS writes, sum(fsyncs) AS fsyncs FROM pg_stat_io WHERE object = 'relation' \gset io_sum_shared_before_ +SELECT sum(writes) AS writes, sum(fsyncs) AS fsyncs + FROM pg_my_stat_io + WHERE object = 'relation' \gset my_io_sum_shared_before_ CREATE TABLE test_io_shared(a int); INSERT INTO test_io_shared SELECT i FROM generate_series(1,100)i; SELECT pg_stat_force_next_flush(); SELECT sum(extends) AS io_sum_shared_after_extends FROM pg_stat_io WHERE context = 'normal' AND object = 'relation' \gset SELECT :io_sum_shared_after_extends > :io_sum_shared_before_extends; +SELECT sum(extends) AS my_io_sum_shared_after_extends + FROM pg_my_stat_io WHERE context = 'normal' AND object = 'relation' \gset +SELECT :my_io_sum_shared_after_extends > :my_io_sum_shared_before_extends; -- After a checkpoint, there should be some additional IOCONTEXT_NORMAL writes --- and fsyncs. +-- and fsyncs in the global stats (not for the backend). -- See comment above for rationale for two explicit CHECKPOINTs. CHECKPOINT; CHECKPOINT; @@ -630,7 +638,13 @@ SELECT sum(writes) AS writes, sum(fsyncs) AS fsyncs SELECT :io_sum_shared_after_writes > :io_sum_shared_before_writes; SELECT current_setting('fsync') = 'off' OR :io_sum_shared_after_fsyncs > :io_sum_shared_before_fsyncs; - +SELECT sum(writes) AS writes, sum(fsyncs) AS fsyncs + FROM pg_my_stat_io + WHERE object = 'relation' \gset my_io_sum_shared_after_ +SELECT :my_io_sum_shared_after_writes >= :my_io_sum_shared_before_writes; +SELECT current_setting('fsync') = 'off' + OR (:my_io_sum_shared_after_fsyncs = :my_io_sum_shared_before_fsyncs + AND :my_io_sum_shared_after_fsyncs= 0); -- Change the tablespace so that the table is rewritten directly, then SELECT -- from it to cause it to be read back into shared buffers. SELECT sum(reads) AS io_sum_shared_before_reads @@ -762,10 +776,15 @@ SELECT :io_sum_bulkwrite_strategy_extends_after > :io_sum_bulkwrite_strategy_ext SELECT pg_stat_have_stats('io', 0, 0); SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS io_stats_pre_reset FROM pg_stat_io \gset +SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS my_io_stats_pre_reset + FROM pg_my_stat_io \gset SELECT pg_stat_reset_shared('io'); SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS io_stats_post_reset FROM pg_stat_io \gset SELECT :io_stats_post_reset < :io_stats_pre_reset; +SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS my_io_stats_post_reset + FROM pg_my_stat_io \gset +SELECT :my_io_stats_post_reset < :my_io_stats_pre_reset; -- test BRIN index doesn't block HOT update -- 2.34.1 --pJYTS7leEEf4d5MD Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v4-0002-Add-pg_stat_get_backend_io.patch" ^ permalink raw reply [nested|flat] 25+ messages in thread
* [PATCH v1 1/2] per backend I/O statistics @ 2024-08-22 15:16 Bertrand Drouvot <[email protected]> 0 siblings, 0 replies; 25+ messages in thread From: Bertrand Drouvot @ 2024-08-22 15:16 UTC (permalink / raw) While pg_stat_io provides cluster-wide I/O statistics, this commit adds a new pg_my_stat_io view to display "my" backend I/O statistics. The KIND_IO stats are still "fixed amount" ones as the maximum number of backend is fixed. The statistics snapshot is made for the global stats (the aggregated ones) and for my backend stats. The snapshot is not build for all the backend stats (that could be memory expensive depending of the number of max connections and given the fact that PgStat_IO is 16K bytes long). A subsequent commit will add a new pg_stat_get_backend_io() function to be able to retrieve the I/O statistics for a given backend pid. Bump catalog version. --- doc/src/sgml/config.sgml | 4 +- doc/src/sgml/monitoring.sgml | 28 +++++++++ src/backend/catalog/system_views.sql | 24 +++++++- src/backend/utils/activity/pgstat.c | 10 ++- src/backend/utils/activity/pgstat_io.c | 75 +++++++++++++++++++---- src/backend/utils/activity/pgstat_shmem.c | 4 +- src/backend/utils/adt/pgstatfuncs.c | 18 +++++- src/include/catalog/catversion.h | 2 +- src/include/catalog/pg_proc.dat | 8 +-- src/include/pgstat.h | 8 ++- src/include/utils/pgstat_internal.h | 13 ++-- src/test/regress/expected/rules.out | 21 ++++++- src/test/regress/expected/stats.out | 44 ++++++++++++- src/test/regress/sql/stats.sql | 25 +++++++- 14 files changed, 245 insertions(+), 39 deletions(-) 10.7% doc/src/sgml/ 4.1% src/backend/catalog/ 30.9% src/backend/utils/activity/ 5.2% src/backend/utils/adt/ 7.9% src/include/catalog/ 3.4% src/include/utils/ 22.0% src/test/regress/expected/ 12.7% src/test/regress/sql/ diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 0aec11f443..2a59b97093 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -8331,7 +8331,9 @@ COPY postgres_log FROM '/full/path/to/logfile.csv' WITH csv; displayed in <link linkend="monitoring-pg-stat-database-view"> <structname>pg_stat_database</structname></link>, <link linkend="monitoring-pg-stat-io-view"> - <structname>pg_stat_io</structname></link>, in the output of + <structname>pg_stat_io</structname></link>, + <link linkend="monitoring-pg-my-stat-io-view"> + <structname>pg_my_stat_io</structname></link>, in the output of <xref linkend="sql-explain"/> when the <literal>BUFFERS</literal> option is used, in the output of <xref linkend="sql-vacuum"/> when the <literal>VERBOSE</literal> option is used, by autovacuum diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml index 55417a6fa9..27d2548d61 100644 --- a/doc/src/sgml/monitoring.sgml +++ b/doc/src/sgml/monitoring.sgml @@ -488,6 +488,16 @@ postgres 27093 0.0 0.0 30096 2752 ? Ss 11:34 0:00 postgres: ser </entry> </row> + <row> + <entry><structname>pg_my_stat_io</structname><indexterm><primary>pg_my_stat_io</primary></indexterm></entry> + <entry> + One row for each combination of context and target object containing + my backend I/O statistics. + See <link linkend="monitoring-pg-my-stat-io-view"> + <structname>pg_my_stat_io</structname></link> for details. + </entry> + </row> + <row> <entry><structname>pg_stat_replication_slots</structname><indexterm><primary>pg_stat_replication_slots</primary></indexterm></entry> <entry>One row per replication slot, showing statistics about the @@ -2875,6 +2885,24 @@ description | Waiting for a newly initialized WAL file to reach durable storage + </sect2> + + <sect2 id="monitoring-pg-my-stat-io-view"> + <title><structname>pg_my_stat_io</structname></title> + + <indexterm> + <primary>pg_my_stat_io</primary> + </indexterm> + + <para> + The <structname>pg_my_stat_io</structname> view will contain one row for each + combination of target I/O object and I/O context, showing + my backend I/O statistics. Combinations which do not make sense are + omitted. The fields are exactly the same as the ones in the <link + linkend="monitoring-pg-stat-io-view"> <structname>pg_stat_io</structname></link> + view. + </para> + </sect2> <sect2 id="monitoring-pg-stat-bgwriter-view"> diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index 19cabc9a47..7fa050893a 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1166,7 +1166,29 @@ SELECT b.fsyncs, b.fsync_time, b.stats_reset -FROM pg_stat_get_io() b; +FROM pg_stat_get_io(true) b; + +CREATE VIEW pg_my_stat_io AS +SELECT + b.backend_type, + b.object, + b.context, + b.reads, + b.read_time, + b.writes, + b.write_time, + b.writebacks, + b.writeback_time, + b.extends, + b.extend_time, + b.op_bytes, + b.hits, + b.evictions, + b.reuses, + b.fsyncs, + b.fsync_time, + b.stats_reset +FROM pg_stat_get_io(false) b; CREATE VIEW pg_stat_wal AS SELECT diff --git a/src/backend/utils/activity/pgstat.c b/src/backend/utils/activity/pgstat.c index b2ca3f39b7..b0905e6bf1 100644 --- a/src/backend/utils/activity/pgstat.c +++ b/src/backend/utils/activity/pgstat.c @@ -406,10 +406,11 @@ static const PgStat_KindInfo pgstat_kind_builtin_infos[PGSTAT_KIND_BUILTIN_SIZE] .fixed_amount = true, - .snapshot_ctl_off = offsetof(PgStat_Snapshot, io), + .snapshot_ctl_off = offsetof(PgStat_Snapshot, global_io), .shared_ctl_off = offsetof(PgStat_ShmemControl, io), - .shared_data_off = offsetof(PgStatShared_IO, stats), - .shared_data_len = sizeof(((PgStatShared_IO *) 0)->stats), + /* [de-]serialize global_stats only */ + .shared_data_off = offsetof(PgStatShared_IO, global_stats), + .shared_data_len = sizeof(((PgStatShared_IO *) 0)->global_stats), .init_shmem_cb = pgstat_io_init_shmem_cb, .reset_all_cb = pgstat_io_reset_all_cb, @@ -580,6 +581,9 @@ pgstat_shutdown_hook(int code, Datum arg) pgstat_report_stat(true); + /* reset the pgstat_io counter related to this proc number */ + pgstat_reset_io_counter_internal(MyProcNumber, GetCurrentTimestamp()); + /* there shouldn't be any pending changes left */ Assert(dlist_is_empty(&pgStatPending)); dlist_init(&pgStatPending); diff --git a/src/backend/utils/activity/pgstat_io.c b/src/backend/utils/activity/pgstat_io.c index 8af55989ee..43280f4892 100644 --- a/src/backend/utils/activity/pgstat_io.c +++ b/src/backend/utils/activity/pgstat_io.c @@ -31,6 +31,7 @@ typedef struct PgStat_PendingIO static PgStat_PendingIO PendingIOStats; bool have_iostats = false; +void pgstat_reset_io_counter_internal(int index, TimestampTz ts); /* * Check that stats have not been counted for any combination of IOObject, @@ -154,11 +155,19 @@ pgstat_count_io_op_time(IOObject io_object, IOContext io_context, IOOp io_op, } PgStat_IO * -pgstat_fetch_stat_io(void) +pgstat_fetch_global_stat_io(void) { pgstat_snapshot_fixed(PGSTAT_KIND_IO); - return &pgStatLocal.snapshot.io; + return &pgStatLocal.snapshot.global_io; +} + +PgStat_IO * +pgstat_fetch_my_stat_io(void) +{ + pgstat_snapshot_fixed(PGSTAT_KIND_IO); + + return &pgStatLocal.snapshot.my_io; } /* @@ -174,13 +183,16 @@ pgstat_flush_io(bool nowait) { LWLock *bktype_lock; PgStat_BktypeIO *bktype_shstats; + PgStat_BktypeIO *global_bktype_shstats; if (!have_iostats) return false; bktype_lock = &pgStatLocal.shmem->io.locks[MyBackendType]; bktype_shstats = - &pgStatLocal.shmem->io.stats.stats[MyBackendType]; + &pgStatLocal.shmem->io.stats[MyProcNumber].stats[MyBackendType]; + global_bktype_shstats = + &pgStatLocal.shmem->io.global_stats.stats[MyBackendType]; if (!nowait) LWLockAcquire(bktype_lock, LW_EXCLUSIVE); @@ -194,19 +206,28 @@ pgstat_flush_io(bool nowait) for (int io_op = 0; io_op < IOOP_NUM_TYPES; io_op++) { instr_time time; + PgStat_Counter counter; + + counter = PendingIOStats.counts[io_object][io_context][io_op]; + + bktype_shstats->counts[io_object][io_context][io_op] += counter; - bktype_shstats->counts[io_object][io_context][io_op] += - PendingIOStats.counts[io_object][io_context][io_op]; + global_bktype_shstats->counts[io_object][io_context][io_op] += + counter; time = PendingIOStats.pending_times[io_object][io_context][io_op]; bktype_shstats->times[io_object][io_context][io_op] += INSTR_TIME_GET_MICROSEC(time); + + global_bktype_shstats->times[io_object][io_context][io_op] += + INSTR_TIME_GET_MICROSEC(time); } } } Assert(pgstat_bktype_io_stats_valid(bktype_shstats, MyBackendType)); + Assert(pgstat_bktype_io_stats_valid(global_bktype_shstats, MyBackendType)); LWLockRelease(bktype_lock); @@ -260,13 +281,38 @@ pgstat_io_init_shmem_cb(void *stats) LWLockInitialize(&stat_shmem->locks[i], LWTRANCHE_PGSTATS_DATA); } +void +pgstat_reset_io_counter_internal(int index, TimestampTz ts) +{ + for (int i = 0; i < BACKEND_NUM_TYPES; i++) + { + LWLock *bktype_lock = &pgStatLocal.shmem->io.locks[i]; + PgStat_BktypeIO *bktype_shstats = &pgStatLocal.shmem->io.stats[index].stats[i]; + + LWLockAcquire(bktype_lock, LW_EXCLUSIVE); + + /* + * Use the lock in the first BackendType's PgStat_BktypeIO to protect + * the reset timestamp as well. + */ + if (i == 0) + pgStatLocal.shmem->io.stats[index].stat_reset_timestamp = ts; + + memset(bktype_shstats, 0, sizeof(*bktype_shstats)); + LWLockRelease(bktype_lock); + } +} + void pgstat_io_reset_all_cb(TimestampTz ts) { + for (int i = 0; i < NumProcStatSlots; i++) + pgstat_reset_io_counter_internal(i, ts); + for (int i = 0; i < BACKEND_NUM_TYPES; i++) { LWLock *bktype_lock = &pgStatLocal.shmem->io.locks[i]; - PgStat_BktypeIO *bktype_shstats = &pgStatLocal.shmem->io.stats.stats[i]; + PgStat_BktypeIO *bktype_shstats = &pgStatLocal.shmem->io.global_stats.stats[i]; LWLockAcquire(bktype_lock, LW_EXCLUSIVE); @@ -275,7 +321,7 @@ pgstat_io_reset_all_cb(TimestampTz ts) * the reset timestamp as well. */ if (i == 0) - pgStatLocal.shmem->io.stats.stat_reset_timestamp = ts; + pgStatLocal.shmem->io.global_stats.stat_reset_timestamp = ts; memset(bktype_shstats, 0, sizeof(*bktype_shstats)); LWLockRelease(bktype_lock); @@ -288,8 +334,10 @@ pgstat_io_snapshot_cb(void) for (int i = 0; i < BACKEND_NUM_TYPES; i++) { LWLock *bktype_lock = &pgStatLocal.shmem->io.locks[i]; - PgStat_BktypeIO *bktype_shstats = &pgStatLocal.shmem->io.stats.stats[i]; - PgStat_BktypeIO *bktype_snap = &pgStatLocal.snapshot.io.stats[i]; + PgStat_BktypeIO *bktype_global_shstats = &pgStatLocal.shmem->io.global_stats.stats[i]; + PgStat_BktypeIO *bktype_global_snap = &pgStatLocal.snapshot.global_io.stats[i]; + PgStat_BktypeIO *bktype_shstats = &pgStatLocal.shmem->io.stats[MyProcNumber].stats[i]; + PgStat_BktypeIO *bktype_snap = &pgStatLocal.snapshot.my_io.stats[i]; LWLockAcquire(bktype_lock, LW_SHARED); @@ -298,10 +346,15 @@ pgstat_io_snapshot_cb(void) * the reset timestamp as well. */ if (i == 0) - pgStatLocal.snapshot.io.stat_reset_timestamp = - pgStatLocal.shmem->io.stats.stat_reset_timestamp; + { + pgStatLocal.snapshot.global_io.stat_reset_timestamp = + pgStatLocal.shmem->io.global_stats.stat_reset_timestamp; + pgStatLocal.snapshot.my_io.stat_reset_timestamp = + pgStatLocal.shmem->io.stats[MyProcNumber].stat_reset_timestamp; + } /* using struct assignment due to better type safety */ + *bktype_global_snap = *bktype_global_shstats; *bktype_snap = *bktype_shstats; LWLockRelease(bktype_lock); } diff --git a/src/backend/utils/activity/pgstat_shmem.c b/src/backend/utils/activity/pgstat_shmem.c index ec93bf6902..7cf1c64f9e 100644 --- a/src/backend/utils/activity/pgstat_shmem.c +++ b/src/backend/utils/activity/pgstat_shmem.c @@ -128,7 +128,7 @@ StatsShmemSize(void) { Size sz; - sz = MAXALIGN(sizeof(PgStat_ShmemControl)); + sz = MAXALIGN(sizeof(PgStat_ShmemControl) + mul_size(sizeof(PgStat_IO), NumProcStatSlots)); sz = add_size(sz, pgstat_dsa_init_size()); /* Add shared memory for all the custom fixed-numbered statistics */ @@ -172,7 +172,7 @@ StatsShmemInit(void) Assert(!found); /* the allocation of pgStatLocal.shmem itself */ - p += MAXALIGN(sizeof(PgStat_ShmemControl)); + p += MAXALIGN(sizeof(PgStat_ShmemControl) + mul_size(sizeof(PgStat_IO), NumProcStatSlots)); /* * Create a small dsa allocation in plain shared memory. This is diff --git a/src/backend/utils/adt/pgstatfuncs.c b/src/backend/utils/adt/pgstatfuncs.c index 3221137123..58e321f421 100644 --- a/src/backend/utils/adt/pgstatfuncs.c +++ b/src/backend/utils/adt/pgstatfuncs.c @@ -1359,18 +1359,30 @@ pg_stat_get_io(PG_FUNCTION_ARGS) ReturnSetInfo *rsinfo; PgStat_IO *backends_io_stats; Datum reset_time; + bool is_global; + + is_global = PG_GETARG_BOOL(0); InitMaterializedSRF(fcinfo, 0); rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; - backends_io_stats = pgstat_fetch_stat_io(); + if (is_global) + backends_io_stats = pgstat_fetch_global_stat_io(); + else + backends_io_stats = pgstat_fetch_my_stat_io(); reset_time = TimestampTzGetDatum(backends_io_stats->stat_reset_timestamp); for (int bktype = 0; bktype < BACKEND_NUM_TYPES; bktype++) { - Datum bktype_desc = CStringGetTextDatum(GetBackendTypeDesc(bktype)); - PgStat_BktypeIO *bktype_stats = &backends_io_stats->stats[bktype]; + Datum bktype_desc; + PgStat_BktypeIO *bktype_stats; + + if (!is_global && bktype != MyBackendType) + continue; + + bktype_desc = CStringGetTextDatum(GetBackendTypeDesc(bktype)); + bktype_stats = &backends_io_stats->stats[bktype]; /* * In Assert builds, we can afford an extra loop through all of the diff --git a/src/include/catalog/catversion.h b/src/include/catalog/catversion.h index 1980d492c3..fbee0db2eb 100644 --- a/src/include/catalog/catversion.h +++ b/src/include/catalog/catversion.h @@ -57,6 +57,6 @@ */ /* yyyymmddN */ -#define CATALOG_VERSION_NO 202408301 +#define CATALOG_VERSION_NO 202409021 #endif diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index 85f42be1b3..1184341fc8 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -5825,10 +5825,10 @@ { oid => '6214', descr => 'statistics: per backend type IO statistics', proname => 'pg_stat_get_io', prorows => '30', proretset => 't', provolatile => 'v', proparallel => 'r', prorettype => 'record', - proargtypes => '', - proallargtypes => '{text,text,text,int8,float8,int8,float8,int8,float8,int8,float8,int8,int8,int8,int8,int8,float8,timestamptz}', - proargmodes => '{o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o}', - proargnames => '{backend_type,object,context,reads,read_time,writes,write_time,writebacks,writeback_time,extends,extend_time,op_bytes,hits,evictions,reuses,fsyncs,fsync_time,stats_reset}', + proargtypes => 'bool', + proallargtypes => '{bool,text,text,text,int8,float8,int8,float8,int8,float8,int8,float8,int8,int8,int8,int8,int8,float8,timestamptz}', + proargmodes => '{i,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o}', + proargnames => '{is_global,backend_type,object,context,reads,read_time,writes,write_time,writebacks,writeback_time,extends,extend_time,op_bytes,hits,evictions,reuses,fsyncs,fsync_time,stats_reset}', prosrc => 'pg_stat_get_io' }, { oid => '1136', descr => 'statistics: information about WAL activity', diff --git a/src/include/pgstat.h b/src/include/pgstat.h index f63159c55c..9ec4053a3e 100644 --- a/src/include/pgstat.h +++ b/src/include/pgstat.h @@ -15,6 +15,7 @@ #include "datatype/timestamp.h" #include "portability/instr_time.h" #include "postmaster/pgarch.h" /* for MAX_XFN_CHARS */ +#include "storage/proc.h" #include "utils/backend_progress.h" /* for backward compatibility */ #include "utils/backend_status.h" /* for backward compatibility */ #include "utils/relcache.h" @@ -75,6 +76,8 @@ */ #define PGSTAT_KIND_EXPERIMENTAL 128 +#define NumProcStatSlots (MaxBackends + NUM_AUXILIARY_PROCS) + static inline bool pgstat_is_kind_builtin(PgStat_Kind kind) { @@ -556,7 +559,8 @@ extern instr_time pgstat_prepare_io_time(bool track_io_guc); extern void pgstat_count_io_op_time(IOObject io_object, IOContext io_context, IOOp io_op, instr_time start_time, uint32 cnt); -extern PgStat_IO *pgstat_fetch_stat_io(void); +extern PgStat_IO *pgstat_fetch_global_stat_io(void); +extern PgStat_IO *pgstat_fetch_my_stat_io(void); extern const char *pgstat_get_io_context_name(IOContext io_context); extern const char *pgstat_get_io_object_name(IOObject io_object); @@ -565,7 +569,7 @@ extern bool pgstat_tracks_io_object(BackendType bktype, IOObject io_object, IOContext io_context); extern bool pgstat_tracks_io_op(BackendType bktype, IOObject io_object, IOContext io_context, IOOp io_op); - +extern void pgstat_reset_io_counter_internal(int index, TimestampTz ts); /* * Functions in pgstat_database.c diff --git a/src/include/utils/pgstat_internal.h b/src/include/utils/pgstat_internal.h index fb132e439d..1eeb8aa107 100644 --- a/src/include/utils/pgstat_internal.h +++ b/src/include/utils/pgstat_internal.h @@ -347,11 +347,12 @@ typedef struct PgStatShared_Checkpointer typedef struct PgStatShared_IO { /* - * locks[i] protects stats.stats[i]. locks[0] also protects - * stats.stat_reset_timestamp. + * locks[i] protects global_stats.stats[i]. locks[0] also protects + * global_stats.stat_reset_timestamp. */ LWLock locks[BACKEND_NUM_TYPES]; - PgStat_IO stats; + PgStat_IO global_stats; + PgStat_IO stats[FLEXIBLE_ARRAY_MEMBER]; } PgStatShared_IO; typedef struct PgStatShared_SLRU @@ -444,7 +445,6 @@ typedef struct PgStat_ShmemControl PgStatShared_Archiver archiver; PgStatShared_BgWriter bgwriter; PgStatShared_Checkpointer checkpointer; - PgStatShared_IO io; PgStatShared_SLRU slru; PgStatShared_Wal wal; @@ -454,6 +454,8 @@ typedef struct PgStat_ShmemControl */ void *custom_data[PGSTAT_KIND_CUSTOM_SIZE]; + /* has to be at the end due to FLEXIBLE_ARRAY_MEMBER */ + PgStatShared_IO io; } PgStat_ShmemControl; @@ -475,7 +477,8 @@ typedef struct PgStat_Snapshot PgStat_CheckpointerStats checkpointer; - PgStat_IO io; + PgStat_IO my_io; + PgStat_IO global_io; PgStat_SLRUStats slru[SLRU_NUM_ELEMENTS]; diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out index 862433ee52..30dcb80f7b 100644 --- a/src/test/regress/expected/rules.out +++ b/src/test/regress/expected/rules.out @@ -1398,6 +1398,25 @@ pg_matviews| SELECT n.nspname AS schemaname, LEFT JOIN pg_namespace n ON ((n.oid = c.relnamespace))) LEFT JOIN pg_tablespace t ON ((t.oid = c.reltablespace))) WHERE (c.relkind = 'm'::"char"); +pg_my_stat_io| SELECT backend_type, + object, + context, + reads, + read_time, + writes, + write_time, + writebacks, + writeback_time, + extends, + extend_time, + op_bytes, + hits, + evictions, + reuses, + fsyncs, + fsync_time, + stats_reset + FROM pg_stat_get_io(false) b(backend_type, object, context, reads, read_time, writes, write_time, writebacks, writeback_time, extends, extend_time, op_bytes, hits, evictions, reuses, fsyncs, fsync_time, stats_reset); pg_policies| SELECT n.nspname AS schemaname, c.relname AS tablename, pol.polname AS policyname, @@ -1902,7 +1921,7 @@ pg_stat_io| SELECT backend_type, fsyncs, fsync_time, stats_reset - FROM pg_stat_get_io() b(backend_type, object, context, reads, read_time, writes, write_time, writebacks, writeback_time, extends, extend_time, op_bytes, hits, evictions, reuses, fsyncs, fsync_time, stats_reset); + FROM pg_stat_get_io(true) b(backend_type, object, context, reads, read_time, writes, write_time, writebacks, writeback_time, extends, extend_time, op_bytes, hits, evictions, reuses, fsyncs, fsync_time, stats_reset); pg_stat_progress_analyze| SELECT s.pid, s.datid, d.datname, diff --git a/src/test/regress/expected/stats.out b/src/test/regress/expected/stats.out index 6e08898b18..c489e528e0 100644 --- a/src/test/regress/expected/stats.out +++ b/src/test/regress/expected/stats.out @@ -1249,7 +1249,7 @@ SELECT pg_stat_get_subscription_stats(NULL); (1 row) --- Test that the following operations are tracked in pg_stat_io: +-- Test that the following operations are tracked in pg_[my]_stat_io: -- - reads of target blocks into shared buffers -- - writes of shared buffers to permanent storage -- - extends of relations using shared buffers @@ -1261,9 +1261,14 @@ SELECT pg_stat_get_subscription_stats(NULL); -- extends. SELECT sum(extends) AS io_sum_shared_before_extends FROM pg_stat_io WHERE context = 'normal' AND object = 'relation' \gset +SELECT sum(extends) AS my_io_sum_shared_before_extends + FROM pg_my_stat_io WHERE context = 'normal' AND object = 'relation' \gset SELECT sum(writes) AS writes, sum(fsyncs) AS fsyncs FROM pg_stat_io WHERE object = 'relation' \gset io_sum_shared_before_ +SELECT sum(writes) AS writes, sum(fsyncs) AS fsyncs + FROM pg_my_stat_io + WHERE object = 'relation' \gset my_io_sum_shared_before_ CREATE TABLE test_io_shared(a int); INSERT INTO test_io_shared SELECT i FROM generate_series(1,100)i; SELECT pg_stat_force_next_flush(); @@ -1280,8 +1285,16 @@ SELECT :io_sum_shared_after_extends > :io_sum_shared_before_extends; t (1 row) +SELECT sum(extends) AS my_io_sum_shared_after_extends + FROM pg_my_stat_io WHERE context = 'normal' AND object = 'relation' \gset +SELECT :my_io_sum_shared_after_extends > :my_io_sum_shared_before_extends; + ?column? +---------- + t +(1 row) + -- After a checkpoint, there should be some additional IOCONTEXT_NORMAL writes --- and fsyncs. +-- and fsyncs in the global stats (not for the backend). -- See comment above for rationale for two explicit CHECKPOINTs. CHECKPOINT; CHECKPOINT; @@ -1301,6 +1314,23 @@ SELECT current_setting('fsync') = 'off' t (1 row) +SELECT sum(writes) AS writes, sum(fsyncs) AS fsyncs + FROM pg_my_stat_io + WHERE object = 'relation' \gset my_io_sum_shared_after_ +SELECT :my_io_sum_shared_after_writes >= :my_io_sum_shared_before_writes; + ?column? +---------- + t +(1 row) + +SELECT current_setting('fsync') = 'off' + OR (:my_io_sum_shared_after_fsyncs = :my_io_sum_shared_before_fsyncs + AND :my_io_sum_shared_after_fsyncs= 0); + ?column? +---------- + t +(1 row) + -- Change the tablespace so that the table is rewritten directly, then SELECT -- from it to cause it to be read back into shared buffers. SELECT sum(reads) AS io_sum_shared_before_reads @@ -1521,6 +1551,8 @@ SELECT pg_stat_have_stats('io', 0, 0); SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS io_stats_pre_reset FROM pg_stat_io \gset +SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS my_io_stats_pre_reset + FROM pg_my_stat_io \gset SELECT pg_stat_reset_shared('io'); pg_stat_reset_shared ---------------------- @@ -1535,6 +1567,14 @@ SELECT :io_stats_post_reset < :io_stats_pre_reset; t (1 row) +SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS my_io_stats_post_reset + FROM pg_my_stat_io \gset +SELECT :my_io_stats_post_reset < :my_io_stats_pre_reset; + ?column? +---------- + t +(1 row) + -- test BRIN index doesn't block HOT update CREATE TABLE brin_hot ( id integer PRIMARY KEY, diff --git a/src/test/regress/sql/stats.sql b/src/test/regress/sql/stats.sql index d8ac0d06f4..c95cb71652 100644 --- a/src/test/regress/sql/stats.sql +++ b/src/test/regress/sql/stats.sql @@ -595,7 +595,7 @@ SELECT pg_stat_get_replication_slot(NULL); SELECT pg_stat_get_subscription_stats(NULL); --- Test that the following operations are tracked in pg_stat_io: +-- Test that the following operations are tracked in pg_[my]_stat_io: -- - reads of target blocks into shared buffers -- - writes of shared buffers to permanent storage -- - extends of relations using shared buffers @@ -609,18 +609,26 @@ SELECT pg_stat_get_subscription_stats(NULL); -- extends. SELECT sum(extends) AS io_sum_shared_before_extends FROM pg_stat_io WHERE context = 'normal' AND object = 'relation' \gset +SELECT sum(extends) AS my_io_sum_shared_before_extends + FROM pg_my_stat_io WHERE context = 'normal' AND object = 'relation' \gset SELECT sum(writes) AS writes, sum(fsyncs) AS fsyncs FROM pg_stat_io WHERE object = 'relation' \gset io_sum_shared_before_ +SELECT sum(writes) AS writes, sum(fsyncs) AS fsyncs + FROM pg_my_stat_io + WHERE object = 'relation' \gset my_io_sum_shared_before_ CREATE TABLE test_io_shared(a int); INSERT INTO test_io_shared SELECT i FROM generate_series(1,100)i; SELECT pg_stat_force_next_flush(); SELECT sum(extends) AS io_sum_shared_after_extends FROM pg_stat_io WHERE context = 'normal' AND object = 'relation' \gset SELECT :io_sum_shared_after_extends > :io_sum_shared_before_extends; +SELECT sum(extends) AS my_io_sum_shared_after_extends + FROM pg_my_stat_io WHERE context = 'normal' AND object = 'relation' \gset +SELECT :my_io_sum_shared_after_extends > :my_io_sum_shared_before_extends; -- After a checkpoint, there should be some additional IOCONTEXT_NORMAL writes --- and fsyncs. +-- and fsyncs in the global stats (not for the backend). -- See comment above for rationale for two explicit CHECKPOINTs. CHECKPOINT; CHECKPOINT; @@ -630,7 +638,13 @@ SELECT sum(writes) AS writes, sum(fsyncs) AS fsyncs SELECT :io_sum_shared_after_writes > :io_sum_shared_before_writes; SELECT current_setting('fsync') = 'off' OR :io_sum_shared_after_fsyncs > :io_sum_shared_before_fsyncs; - +SELECT sum(writes) AS writes, sum(fsyncs) AS fsyncs + FROM pg_my_stat_io + WHERE object = 'relation' \gset my_io_sum_shared_after_ +SELECT :my_io_sum_shared_after_writes >= :my_io_sum_shared_before_writes; +SELECT current_setting('fsync') = 'off' + OR (:my_io_sum_shared_after_fsyncs = :my_io_sum_shared_before_fsyncs + AND :my_io_sum_shared_after_fsyncs= 0); -- Change the tablespace so that the table is rewritten directly, then SELECT -- from it to cause it to be read back into shared buffers. SELECT sum(reads) AS io_sum_shared_before_reads @@ -762,10 +776,15 @@ SELECT :io_sum_bulkwrite_strategy_extends_after > :io_sum_bulkwrite_strategy_ext SELECT pg_stat_have_stats('io', 0, 0); SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS io_stats_pre_reset FROM pg_stat_io \gset +SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS my_io_stats_pre_reset + FROM pg_my_stat_io \gset SELECT pg_stat_reset_shared('io'); SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS io_stats_post_reset FROM pg_stat_io \gset SELECT :io_stats_post_reset < :io_stats_pre_reset; +SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS my_io_stats_post_reset + FROM pg_my_stat_io \gset +SELECT :my_io_stats_post_reset < :my_io_stats_pre_reset; -- test BRIN index doesn't block HOT update -- 2.34.1 --8G6ZM24Fzj1jtUgQ Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v1-0002-Add-pg_stat_get_backend_io.patch" ^ permalink raw reply [nested|flat] 25+ messages in thread
* [PATCH v9] per backend I/O statistics @ 2024-10-28 12:50 Bertrand Drouvot <[email protected]> 0 siblings, 0 replies; 25+ messages in thread From: Bertrand Drouvot @ 2024-10-28 12:50 UTC (permalink / raw) While pg_stat_io provides cluster-wide I/O statistics, this commit adds the ability to track and display per backend I/O statistics. It adds a new statistics kind and 2 new functions: - pg_stat_reset_backend_stats() to be able to reset the stats for a given backend pid. - pg_stat_get_backend_io() to retrieve I/O statistics for a given backend pid. The new KIND is named PGSTAT_KIND_BACKEND as it could be used in the future to store other statistics (than the I/O ones) per backend. The new KIND is a variable-numbered one and has an automatic cap on the maximum number of entries (as its hash key contains the proc number). There is no need to write the per backend I/O stats to disk (no point to see stats for backends that do not exist anymore after a re-start), so using "write_to_file = false". Note that per backend I/O statistics are not collected for the checkpointer, the background writer, the startup process and the autovacuum launcher as those are already visible in pg_stat_io and there is only one of those. XXX: Bump catalog version needs to be done. --- doc/src/sgml/config.sgml | 8 +- doc/src/sgml/monitoring.sgml | 37 ++++ src/backend/catalog/system_functions.sql | 2 + src/backend/utils/activity/Makefile | 1 + src/backend/utils/activity/backend_status.c | 4 + src/backend/utils/activity/meson.build | 1 + src/backend/utils/activity/pgstat.c | 23 ++- src/backend/utils/activity/pgstat_backend.c | 182 +++++++++++++++++++ src/backend/utils/activity/pgstat_io.c | 25 ++- src/backend/utils/activity/pgstat_relation.c | 2 + src/backend/utils/adt/pgstatfuncs.c | 169 +++++++++++++++++ src/include/catalog/pg_proc.dat | 14 ++ src/include/pgstat.h | 31 +++- src/include/utils/pgstat_internal.h | 14 ++ src/test/regress/expected/stats.out | 72 +++++++- src/test/regress/sql/stats.sql | 38 +++- src/tools/pgindent/typedefs.list | 3 + 17 files changed, 605 insertions(+), 21 deletions(-) 9.9% doc/src/sgml/ 31.3% src/backend/utils/activity/ 22.0% src/backend/utils/adt/ 4.4% src/include/catalog/ 7.0% src/include/ 12.8% src/test/regress/expected/ 11.5% src/test/regress/sql/ diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index e0c8325a39..8afca9b110 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -8403,9 +8403,11 @@ COPY postgres_log FROM '/full/path/to/logfile.csv' WITH csv; displayed in <link linkend="monitoring-pg-stat-database-view"> <structname>pg_stat_database</structname></link>, <link linkend="monitoring-pg-stat-io-view"> - <structname>pg_stat_io</structname></link>, in the output of - <xref linkend="sql-explain"/> when the <literal>BUFFERS</literal> option - is used, in the output of <xref linkend="sql-vacuum"/> when + <structname>pg_stat_io</structname></link>, in the output of the + <link linkend="pg-stat-get-backend-io"> + <function>pg_stat_get_backend_io()</function></link> function, in the + output of <xref linkend="sql-explain"/> when the <literal>BUFFERS</literal> + option is used, in the output of <xref linkend="sql-vacuum"/> when the <literal>VERBOSE</literal> option is used, by autovacuum for auto-vacuums and auto-analyzes, when <xref linkend="guc-log-autovacuum-min-duration"/> is set and by diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml index 840d7f8161..4a9464da61 100644 --- a/doc/src/sgml/monitoring.sgml +++ b/doc/src/sgml/monitoring.sgml @@ -4790,6 +4790,25 @@ description | Waiting for a newly initialized WAL file to reach durable storage </para></entry> </row> + <row> + <entry id="pg-stat-get-backend-io" role="func_table_entry"><para role="func_signature"> + <indexterm> + <primary>pg_stat_get_backend_io</primary> + </indexterm> + <function>pg_stat_get_backend_io</function> ( <type>integer</type> ) + <returnvalue>setof record</returnvalue> + </para> + <para> + Returns I/O statistics about the backend with the specified + process ID. The output fields are exactly the same as the ones in the + <link linkend="monitoring-pg-stat-io-view"> <structname>pg_stat_io</structname></link> + view. The function does not return I/O statistics for the checkpointer, + the background writer, the startup process and the autovacuum launcher + as they are already visible in the <link linkend="monitoring-pg-stat-io-view"> <structname>pg_stat_io</structname></link> + view and there is only one of those. + </para></entry> + </row> + <row> <entry role="func_table_entry"><para role="func_signature"> <indexterm> @@ -4971,6 +4990,24 @@ description | Waiting for a newly initialized WAL file to reach durable storage </para></entry> </row> + <row> + <entry role="func_table_entry"><para role="func_signature"> + <indexterm> + <primary>pg_stat_reset_backend_stats</primary> + </indexterm> + <function>pg_stat_reset_backend_stats</function> ( <type>integer</type> ) + <returnvalue>void</returnvalue> + </para> + <para> + Resets statistics for a single backend with the specified process ID + to zero. + </para> + <para> + This function is restricted to superusers by default, but other users + can be granted EXECUTE to run the function. + </para></entry> + </row> + <row> <entry role="func_table_entry"><para role="func_signature"> <indexterm> diff --git a/src/backend/catalog/system_functions.sql b/src/backend/catalog/system_functions.sql index c51dfca802..14eb99cd47 100644 --- a/src/backend/catalog/system_functions.sql +++ b/src/backend/catalog/system_functions.sql @@ -711,6 +711,8 @@ REVOKE EXECUTE ON FUNCTION pg_stat_reset_single_table_counters(oid) FROM public; REVOKE EXECUTE ON FUNCTION pg_stat_reset_single_function_counters(oid) FROM public; +REVOKE EXECUTE ON FUNCTION pg_stat_reset_backend_stats(integer) FROM public; + REVOKE EXECUTE ON FUNCTION pg_stat_reset_replication_slot(text) FROM public; REVOKE EXECUTE ON FUNCTION pg_stat_have_stats(text, oid, int8) FROM public; diff --git a/src/backend/utils/activity/Makefile b/src/backend/utils/activity/Makefile index b9fd66ea17..24b64a2742 100644 --- a/src/backend/utils/activity/Makefile +++ b/src/backend/utils/activity/Makefile @@ -20,6 +20,7 @@ OBJS = \ backend_status.o \ pgstat.o \ pgstat_archiver.o \ + pgstat_backend.o \ pgstat_bgwriter.o \ pgstat_checkpointer.o \ pgstat_database.o \ diff --git a/src/backend/utils/activity/backend_status.c b/src/backend/utils/activity/backend_status.c index 22c6dc378c..e9c3b87ac2 100644 --- a/src/backend/utils/activity/backend_status.c +++ b/src/backend/utils/activity/backend_status.c @@ -426,6 +426,10 @@ pgstat_bestart(void) PGSTAT_END_WRITE_ACTIVITY(vbeentry); + /* Create the backend statistics entry */ + if (pgstat_tracks_backend_bktype(MyBackendType)) + pgstat_create_backend_stat(MyProcNumber); + /* Update app name to current GUC setting */ if (application_name) pgstat_report_appname(application_name); diff --git a/src/backend/utils/activity/meson.build b/src/backend/utils/activity/meson.build index f73c22905c..380d3dd70c 100644 --- a/src/backend/utils/activity/meson.build +++ b/src/backend/utils/activity/meson.build @@ -5,6 +5,7 @@ backend_sources += files( 'backend_status.c', 'pgstat.c', 'pgstat_archiver.c', + 'pgstat_backend.c', 'pgstat_bgwriter.c', 'pgstat_checkpointer.c', 'pgstat_database.c', diff --git a/src/backend/utils/activity/pgstat.c b/src/backend/utils/activity/pgstat.c index 18b7d9b47d..9dd9d60b00 100644 --- a/src/backend/utils/activity/pgstat.c +++ b/src/backend/utils/activity/pgstat.c @@ -77,6 +77,7 @@ * * Each statistics kind is handled in a dedicated file: * - pgstat_archiver.c + * - pgstat_backend.c * - pgstat_bgwriter.c * - pgstat_checkpointer.c * - pgstat_database.c @@ -358,6 +359,22 @@ static const PgStat_KindInfo pgstat_kind_builtin_infos[PGSTAT_KIND_BUILTIN_SIZE] .reset_timestamp_cb = pgstat_subscription_reset_timestamp_cb, }, + [PGSTAT_KIND_BACKEND] = { + .name = "backend", + + .fixed_amount = false, + .write_to_file = false, + + .accessed_across_databases = true, + + .shared_size = sizeof(PgStatShared_Backend), + .shared_data_off = offsetof(PgStatShared_Backend, stats), + .shared_data_len = sizeof(((PgStatShared_Backend *) 0)->stats), + .pending_size = sizeof(PgStat_BackendPendingIO), + + .flush_pending_cb = pgstat_backend_flush_cb, + .reset_timestamp_cb = pgstat_backend_reset_timestamp_cb, + }, /* stats for fixed-numbered (mostly 1) objects */ @@ -602,6 +619,10 @@ pgstat_shutdown_hook(int code, Datum arg) Assert(dlist_is_empty(&pgStatPending)); dlist_init(&pgStatPending); + /* drop the backend stats entry */ + if (!pgstat_drop_entry(PGSTAT_KIND_BACKEND, InvalidOid, MyProcNumber)) + pgstat_request_entry_refs_gc(); + pgstat_detach_shmem(); #ifdef USE_ASSERT_CHECKING @@ -768,7 +789,7 @@ pgstat_report_stat(bool force) partial_flush = false; - /* flush database / relation / function / ... stats */ + /* flush database / relation / function / backend / ... stats */ partial_flush |= pgstat_flush_pending_entries(nowait); /* flush of fixed-numbered stats */ diff --git a/src/backend/utils/activity/pgstat_backend.c b/src/backend/utils/activity/pgstat_backend.c new file mode 100644 index 0000000000..e2d83024c2 --- /dev/null +++ b/src/backend/utils/activity/pgstat_backend.c @@ -0,0 +1,182 @@ +/* ------------------------------------------------------------------------- + * + * pgstat_backend.c + * Implementation of backend statistics. + * + * This file contains the implementation of backend statistics. It is kept + * separate from pgstat.c to enforce the line between the statistics access / + * storage implementation and the details about individual types of statistics. + * + * Copyright (c) 2024, PostgreSQL Global Development Group + * + * IDENTIFICATION + * src/backend/utils/activity/pgstat_backend.c + * ------------------------------------------------------------------------- + */ + +#include "postgres.h" + +#include "utils/pgstat_internal.h" + +/* + * Returns backend's IO stats. + */ +PgStat_Backend * +pgstat_fetch_proc_stat_io(ProcNumber procNumber) +{ + PgStat_Backend *backend_entry; + + backend_entry = (PgStat_Backend *) pgstat_fetch_entry(PGSTAT_KIND_BACKEND, + InvalidOid, procNumber); + + return backend_entry; +} + +/* + * Flush out locally pending backend statistics + * + * If no stats have been recorded, this function returns false. + */ +bool +pgstat_backend_flush_cb(PgStat_EntryRef *entry_ref, bool nowait) +{ + PgStatShared_Backend *shbackendioent; + PgStat_BackendPendingIO *pendingent; + PgStat_BktypeIO *bktype_shstats; + + if (!pgstat_lock_entry(entry_ref, nowait)) + return false; + + shbackendioent = (PgStatShared_Backend *) entry_ref->shared_stats; + bktype_shstats = &shbackendioent->stats.stats; + pendingent = (PgStat_BackendPendingIO *) entry_ref->pending; + + for (int io_object = 0; io_object < IOOBJECT_NUM_TYPES; io_object++) + { + for (int io_context = 0; io_context < IOCONTEXT_NUM_TYPES; io_context++) + { + for (int io_op = 0; io_op < IOOP_NUM_TYPES; io_op++) + { + instr_time time; + + bktype_shstats->counts[io_object][io_context][io_op] += + pendingent->counts[io_object][io_context][io_op]; + + time = pendingent->pending_times[io_object][io_context][io_op]; + + bktype_shstats->times[io_object][io_context][io_op] += + INSTR_TIME_GET_MICROSEC(time); + } + } + } + + pgstat_unlock_entry(entry_ref); + + return true; +} + +/* + * Simpler wrapper of pgstat_backend_flush_cb() + */ +void +pgstat_flush_backend(bool nowait) +{ + if (pgstat_tracks_backend_bktype(MyBackendType)) + { + PgStat_EntryRef *entry_ref; + + entry_ref = pgstat_get_entry_ref(PGSTAT_KIND_BACKEND, InvalidOid, + MyProcNumber, false, NULL); + (void) pgstat_backend_flush_cb(entry_ref, nowait); + } +} + +/* + * Create the backend statistics entry for procnum. + */ +void +pgstat_create_backend_stat(ProcNumber procnum) +{ + PgStat_EntryRef *entry_ref; + PgStatShared_Backend *shstatent; + + entry_ref = pgstat_prep_pending_entry(PGSTAT_KIND_BACKEND, InvalidOid, + procnum, NULL); + + shstatent = (PgStatShared_Backend *) entry_ref->shared_stats; + + /* + * NB: need to accept that there might be stats from an older backend, + * e.g. if we previously used this proc number. + */ + memset(&shstatent->stats, 0, sizeof(shstatent->stats)); +} + +/* + * Find or create a local PgStat_BackendPendingIO entry for procnum. + */ +PgStat_BackendPendingIO * +pgstat_prep_backend_pending(ProcNumber procnum) +{ + PgStat_EntryRef *entry_ref; + + entry_ref = pgstat_prep_pending_entry(PGSTAT_KIND_BACKEND, InvalidOid, + procnum, NULL); + + return entry_ref->pending; +} + +/* + * Backend statistics are not collected for all BackendTypes. + * + * The following BackendTypes do not participate in the backend stats + * subsystem: + * - The same and for the same reasons as in pgstat_tracks_io_bktype(). + * - B_BG_WRITER, B_CHECKPOINTER, B_STARTUP and B_AUTOVAC_LAUNCHER because their + * I/O stats are already visible in pg_stat_io and there is only one of those. + * + * Function returns true if BackendType participates in the backend stats + * subsystem for IO and false if it does not. + * + * When adding a new BackendType, also consider adding relevant restrictions to + * pgstat_tracks_io_object() and pgstat_tracks_io_op(). + */ +bool +pgstat_tracks_backend_bktype(BackendType bktype) +{ + /* + * List every type so that new backend types trigger a warning about + * needing to adjust this switch. + */ + switch (bktype) + { + case B_INVALID: + case B_AUTOVAC_LAUNCHER: + case B_DEAD_END_BACKEND: + case B_ARCHIVER: + case B_LOGGER: + case B_WAL_RECEIVER: + case B_WAL_WRITER: + case B_WAL_SUMMARIZER: + case B_BG_WRITER: + case B_CHECKPOINTER: + case B_STARTUP: + return false; + + case B_AUTOVAC_WORKER: + case B_BACKEND: + case B_BG_WORKER: + case B_STANDALONE_BACKEND: + case B_SLOTSYNC_WORKER: + case B_WAL_SENDER: + return true; + } + + return false; +} + +void +pgstat_backend_reset_timestamp_cb(PgStatShared_Common *header, TimestampTz ts) +{ + ((PgStatShared_Backend *) header)->stats.stat_reset_timestamp = ts; +} diff --git a/src/backend/utils/activity/pgstat_io.c b/src/backend/utils/activity/pgstat_io.c index f9883af2b3..e0c206a453 100644 --- a/src/backend/utils/activity/pgstat_io.c +++ b/src/backend/utils/activity/pgstat_io.c @@ -20,13 +20,7 @@ #include "storage/bufmgr.h" #include "utils/pgstat_internal.h" - -typedef struct PgStat_PendingIO -{ - PgStat_Counter counts[IOOBJECT_NUM_TYPES][IOCONTEXT_NUM_TYPES][IOOP_NUM_TYPES]; - instr_time pending_times[IOOBJECT_NUM_TYPES][IOCONTEXT_NUM_TYPES][IOOP_NUM_TYPES]; -} PgStat_PendingIO; - +typedef PgStat_BackendPendingIO PgStat_PendingIO; static PgStat_PendingIO PendingIOStats; static bool have_iostats = false; @@ -87,6 +81,14 @@ pgstat_count_io_op_n(IOObject io_object, IOContext io_context, IOOp io_op, uint3 Assert((unsigned int) io_op < IOOP_NUM_TYPES); Assert(pgstat_tracks_io_op(MyBackendType, io_object, io_context, io_op)); + if (pgstat_tracks_backend_bktype(MyBackendType)) + { + PgStat_PendingIO *entry_ref; + + entry_ref = pgstat_prep_backend_pending(MyProcNumber); + entry_ref->counts[io_object][io_context][io_op] += cnt; + } + PendingIOStats.counts[io_object][io_context][io_op] += cnt; have_iostats = true; @@ -148,6 +150,15 @@ pgstat_count_io_op_time(IOObject io_object, IOContext io_context, IOOp io_op, INSTR_TIME_ADD(PendingIOStats.pending_times[io_object][io_context][io_op], io_time); + + if (pgstat_tracks_backend_bktype(MyBackendType)) + { + PgStat_PendingIO *entry_ref; + + entry_ref = pgstat_prep_backend_pending(MyProcNumber); + INSTR_TIME_ADD(entry_ref->pending_times[io_object][io_context][io_op], + io_time); + } } pgstat_count_io_op_n(io_object, io_context, io_op, cnt); diff --git a/src/backend/utils/activity/pgstat_relation.c b/src/backend/utils/activity/pgstat_relation.c index faba8b64d2..85e65557bb 100644 --- a/src/backend/utils/activity/pgstat_relation.c +++ b/src/backend/utils/activity/pgstat_relation.c @@ -264,6 +264,7 @@ pgstat_report_vacuum(Oid tableoid, bool shared, * VACUUM command has processed all tables and committed. */ pgstat_flush_io(false); + pgstat_flush_backend(false); } /* @@ -350,6 +351,7 @@ pgstat_report_analyze(Relation rel, /* see pgstat_report_vacuum() */ pgstat_flush_io(false); + pgstat_flush_backend(false); } /* diff --git a/src/backend/utils/adt/pgstatfuncs.c b/src/backend/utils/adt/pgstatfuncs.c index cdf37403e9..b939551d36 100644 --- a/src/backend/utils/adt/pgstatfuncs.c +++ b/src/backend/utils/adt/pgstatfuncs.c @@ -1474,6 +1474,154 @@ pg_stat_get_io(PG_FUNCTION_ARGS) return (Datum) 0; } +Datum +pg_stat_get_backend_io(PG_FUNCTION_ARGS) +{ + ReturnSetInfo *rsinfo; + PgStat_Backend *backend_stats; + Datum bktype_desc; + PgStat_BktypeIO *bktype_stats; + BackendType bktype; + Datum reset_time; + int num_backends = pgstat_fetch_stat_numbackends(); + int curr_backend; + int pid; + PGPROC *proc; + ProcNumber procNumber; + + InitMaterializedSRF(fcinfo, 0); + rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; + + pid = PG_GETARG_INT32(0); + proc = BackendPidGetProc(pid); + + /* + * Could be an auxiliary process but would not report any stats due to + * pgstat_tracks_backend_bktype() anyway. So don't need an extra call to + * AuxiliaryPidGetProc(). + */ + if (!proc) + return (Datum) 0; + + procNumber = GetNumberFromPGProc(proc); + backend_stats = pgstat_fetch_proc_stat_io(procNumber); + + if (!backend_stats) + return (Datum) 0; + + bktype = B_INVALID; + + /* Look for the backend type */ + for (curr_backend = 1; curr_backend <= num_backends; curr_backend++) + { + LocalPgBackendStatus *local_beentry; + PgBackendStatus *beentry; + + /* Get the next one in the list */ + local_beentry = pgstat_get_local_beentry_by_index(curr_backend); + beentry = &local_beentry->backendStatus; + + /* Looking for specific PID, ignore all the others */ + if (beentry->st_procpid != pid) + continue; + + bktype = beentry->st_backendType; + break; + } + + /* Backend is gone */ + if (bktype == B_INVALID) + return (Datum) 0; + + bktype_desc = CStringGetTextDatum(GetBackendTypeDesc(bktype)); + bktype_stats = &backend_stats->stats; + reset_time = TimestampTzGetDatum(backend_stats->stat_reset_timestamp); + + /* + * In Assert builds, we can afford an extra loop through all of the + * counters checking that only expected stats are non-zero, since it keeps + * the non-Assert code cleaner. + */ + Assert(pgstat_bktype_io_stats_valid(bktype_stats, bktype)); + + for (int io_obj = 0; io_obj < IOOBJECT_NUM_TYPES; io_obj++) + { + const char *obj_name = pgstat_get_io_object_name(io_obj); + + for (int io_context = 0; io_context < IOCONTEXT_NUM_TYPES; io_context++) + { + const char *context_name = pgstat_get_io_context_name(io_context); + + Datum values[IO_NUM_COLUMNS] = {0}; + bool nulls[IO_NUM_COLUMNS] = {0}; + + /* + * Some combinations of BackendType, IOObject, and IOContext are + * not valid for any type of IOOp. In such cases, omit the entire + * row from the view. + */ + if (!pgstat_tracks_io_object(bktype, io_obj, io_context)) + continue; + + values[IO_COL_BACKEND_TYPE] = bktype_desc; + values[IO_COL_CONTEXT] = CStringGetTextDatum(context_name); + values[IO_COL_OBJECT] = CStringGetTextDatum(obj_name); + if (backend_stats->stat_reset_timestamp != 0) + values[IO_COL_RESET_TIME] = reset_time; + else + nulls[IO_COL_RESET_TIME] = true; + + /* + * Hard-code this to the value of BLCKSZ for now. Future values + * could include XLOG_BLCKSZ, once WAL IO is tracked, and constant + * multipliers, once non-block-oriented IO (e.g. temporary file + * IO) is tracked. + */ + values[IO_COL_CONVERSION] = Int64GetDatum(BLCKSZ); + + for (int io_op = 0; io_op < IOOP_NUM_TYPES; io_op++) + { + int op_idx = pgstat_get_io_op_index(io_op); + int time_idx = pgstat_get_io_time_index(io_op); + + /* + * Some combinations of BackendType and IOOp, of IOContext and + * IOOp, and of IOObject and IOOp are not tracked. Set these + * cells in the view NULL. + */ + if (pgstat_tracks_io_op(bktype, io_obj, io_context, io_op)) + { + PgStat_Counter count = + bktype_stats->counts[io_obj][io_context][io_op]; + + values[op_idx] = Int64GetDatum(count); + } + else + nulls[op_idx] = true; + + /* not every operation is timed */ + if (time_idx == IO_COL_INVALID) + continue; + + if (!nulls[op_idx]) + { + PgStat_Counter time = + bktype_stats->times[io_obj][io_context][io_op]; + + values[time_idx] = Float8GetDatum(pg_stat_us_to_ms(time)); + } + else + nulls[time_idx] = true; + } + + tuplestore_putvalues(rsinfo->setResult, rsinfo->setDesc, + values, nulls); + } + } + + return (Datum) 0; +} + /* * Returns statistics of WAL activity */ @@ -1779,6 +1927,27 @@ pg_stat_reset_single_function_counters(PG_FUNCTION_ARGS) PG_RETURN_VOID(); } +Datum +pg_stat_reset_backend_stats(PG_FUNCTION_ARGS) +{ + PGPROC *proc; + int backend_pid = PG_GETARG_INT32(0); + + proc = BackendPidGetProc(backend_pid); + + /* + * Could be an auxiliary process but would not report any stats due to + * pgstat_tracks_backend_bktype() anyway. So don't need an extra call to + * AuxiliaryPidGetProc(). + */ + if (!proc) + PG_RETURN_VOID(); + + pgstat_reset(PGSTAT_KIND_BACKEND, InvalidOid, GetNumberFromPGProc(proc)); + + PG_RETURN_VOID(); +} + /* Reset SLRU counters (a specific one or all of them). */ Datum pg_stat_reset_slru(PG_FUNCTION_ARGS) diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index 0f22c21723..437157ffa3 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -5913,6 +5913,15 @@ proargnames => '{backend_type,object,context,reads,read_time,writes,write_time,writebacks,writeback_time,extends,extend_time,op_bytes,hits,evictions,reuses,fsyncs,fsync_time,stats_reset}', prosrc => 'pg_stat_get_io' }, +{ oid => '8806', descr => 'statistics: backend IO statistics', + proname => 'pg_stat_get_backend_io', prorows => '5', proretset => 't', + provolatile => 'v', proparallel => 'r', prorettype => 'record', + proargtypes => 'int4', + proallargtypes => '{int4,text,text,text,int8,float8,int8,float8,int8,float8,int8,float8,int8,int8,int8,int8,int8,float8,timestamptz}', + proargmodes => '{i,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o}', + proargnames => '{backend_pid,backend_type,object,context,reads,read_time,writes,write_time,writebacks,writeback_time,extends,extend_time,op_bytes,hits,evictions,reuses,fsyncs,fsync_time,stats_reset}', + prosrc => 'pg_stat_get_backend_io' }, + { oid => '1136', descr => 'statistics: information about WAL activity', proname => 'pg_stat_get_wal', proisstrict => 'f', provolatile => 's', proparallel => 'r', prorettype => 'record', proargtypes => '', @@ -6052,6 +6061,11 @@ proname => 'pg_stat_reset_single_function_counters', provolatile => 'v', prorettype => 'void', proargtypes => 'oid', prosrc => 'pg_stat_reset_single_function_counters' }, +{ oid => '9987', + descr => 'statistics: reset statistics for a single backend', + proname => 'pg_stat_reset_backend_stats', provolatile => 'v', + prorettype => 'void', proargtypes => 'int4', + prosrc => 'pg_stat_reset_backend_stats' }, { oid => '2307', descr => 'statistics: reset collected statistics for a single SLRU', proname => 'pg_stat_reset_slru', proisstrict => 'f', provolatile => 'v', diff --git a/src/include/pgstat.h b/src/include/pgstat.h index ebfeef2f46..479773cfd2 100644 --- a/src/include/pgstat.h +++ b/src/include/pgstat.h @@ -49,14 +49,15 @@ #define PGSTAT_KIND_FUNCTION 3 /* per-function statistics */ #define PGSTAT_KIND_REPLSLOT 4 /* per-slot statistics */ #define PGSTAT_KIND_SUBSCRIPTION 5 /* per-subscription statistics */ +#define PGSTAT_KIND_BACKEND 6 /* per-backend statistics */ /* stats for fixed-numbered objects */ -#define PGSTAT_KIND_ARCHIVER 6 -#define PGSTAT_KIND_BGWRITER 7 -#define PGSTAT_KIND_CHECKPOINTER 8 -#define PGSTAT_KIND_IO 9 -#define PGSTAT_KIND_SLRU 10 -#define PGSTAT_KIND_WAL 11 +#define PGSTAT_KIND_ARCHIVER 7 +#define PGSTAT_KIND_BGWRITER 8 +#define PGSTAT_KIND_CHECKPOINTER 9 +#define PGSTAT_KIND_IO 10 +#define PGSTAT_KIND_SLRU 11 +#define PGSTAT_KIND_WAL 12 #define PGSTAT_KIND_BUILTIN_MIN PGSTAT_KIND_DATABASE #define PGSTAT_KIND_BUILTIN_MAX PGSTAT_KIND_WAL @@ -362,12 +363,23 @@ typedef struct PgStat_BktypeIO PgStat_Counter times[IOOBJECT_NUM_TYPES][IOCONTEXT_NUM_TYPES][IOOP_NUM_TYPES]; } PgStat_BktypeIO; +typedef struct PgStat_BackendPendingIO +{ + PgStat_Counter counts[IOOBJECT_NUM_TYPES][IOCONTEXT_NUM_TYPES][IOOP_NUM_TYPES]; + instr_time pending_times[IOOBJECT_NUM_TYPES][IOCONTEXT_NUM_TYPES][IOOP_NUM_TYPES]; +} PgStat_BackendPendingIO; + typedef struct PgStat_IO { TimestampTz stat_reset_timestamp; PgStat_BktypeIO stats[BACKEND_NUM_TYPES]; } PgStat_IO; +typedef struct PgStat_Backend +{ + TimestampTz stat_reset_timestamp; + PgStat_BktypeIO stats; +} PgStat_Backend; typedef struct PgStat_StatDBEntry { @@ -549,6 +561,13 @@ extern bool pgstat_have_entry(PgStat_Kind kind, Oid dboid, uint64 objid); extern void pgstat_report_archiver(const char *xlog, bool failed); extern PgStat_ArchiverStats *pgstat_fetch_stat_archiver(void); +/* + * Functions in pgstat_backend.c + */ + +extern PgStat_Backend *pgstat_fetch_proc_stat_io(ProcNumber procNumber); +extern bool pgstat_tracks_backend_bktype(BackendType bktype); +extern void pgstat_create_backend_stat(ProcNumber procnum); /* * Functions in pgstat_bgwriter.c diff --git a/src/include/utils/pgstat_internal.h b/src/include/utils/pgstat_internal.h index 7338bc1e28..811ed9b005 100644 --- a/src/include/utils/pgstat_internal.h +++ b/src/include/utils/pgstat_internal.h @@ -450,6 +450,11 @@ typedef struct PgStatShared_ReplSlot PgStat_StatReplSlotEntry stats; } PgStatShared_ReplSlot; +typedef struct PgStatShared_Backend +{ + PgStatShared_Common header; + PgStat_Backend stats; +} PgStatShared_Backend; /* * Central shared memory entry for the cumulative stats system. @@ -604,6 +609,15 @@ extern void pgstat_archiver_init_shmem_cb(void *stats); extern void pgstat_archiver_reset_all_cb(TimestampTz ts); extern void pgstat_archiver_snapshot_cb(void); +/* + * Functions in pgstat_backend.c + */ + +extern void pgstat_flush_backend(bool nowait); + +extern PgStat_BackendPendingIO *pgstat_prep_backend_pending(ProcNumber procnum); +extern bool pgstat_backend_flush_cb(PgStat_EntryRef *entry_ref, bool nowait); +extern void pgstat_backend_reset_timestamp_cb(PgStatShared_Common *header, TimestampTz ts); /* * Functions in pgstat_bgwriter.c diff --git a/src/test/regress/expected/stats.out b/src/test/regress/expected/stats.out index 56771f83ed..3447e7b75d 100644 --- a/src/test/regress/expected/stats.out +++ b/src/test/regress/expected/stats.out @@ -1249,7 +1249,7 @@ SELECT pg_stat_get_subscription_stats(NULL); (1 row) --- Test that the following operations are tracked in pg_stat_io: +-- Test that the following operations are tracked in pg_stat_io and in backend stats: -- - reads of target blocks into shared buffers -- - writes of shared buffers to permanent storage -- - extends of relations using shared buffers @@ -1259,11 +1259,19 @@ SELECT pg_stat_get_subscription_stats(NULL); -- be sure of the state of shared buffers at the point the test is run. -- Create a regular table and insert some data to generate IOCONTEXT_NORMAL -- extends. +SELECT pid AS checkpointer_pid FROM pg_stat_activity + WHERE backend_type = 'checkpointer' \gset SELECT sum(extends) AS io_sum_shared_before_extends FROM pg_stat_io WHERE context = 'normal' AND object = 'relation' \gset +SELECT sum(extends) AS my_io_sum_shared_before_extends + FROM pg_stat_get_backend_io(pg_backend_pid()) + WHERE context = 'normal' AND object = 'relation' \gset SELECT sum(writes) AS writes, sum(fsyncs) AS fsyncs FROM pg_stat_io WHERE object = 'relation' \gset io_sum_shared_before_ +SELECT sum(writes) AS writes, sum(fsyncs) AS fsyncs + FROM pg_stat_get_backend_io(pg_backend_pid()) + WHERE object = 'relation' \gset my_io_sum_shared_before_ CREATE TABLE test_io_shared(a int); INSERT INTO test_io_shared SELECT i FROM generate_series(1,100)i; SELECT pg_stat_force_next_flush(); @@ -1280,8 +1288,17 @@ SELECT :io_sum_shared_after_extends > :io_sum_shared_before_extends; t (1 row) +SELECT sum(extends) AS my_io_sum_shared_after_extends + FROM pg_stat_get_backend_io(pg_backend_pid()) + WHERE context = 'normal' AND object = 'relation' \gset +SELECT :my_io_sum_shared_after_extends > :my_io_sum_shared_before_extends; + ?column? +---------- + t +(1 row) + -- After a checkpoint, there should be some additional IOCONTEXT_NORMAL writes --- and fsyncs. +-- and fsyncs in the global stats (not for the backend). -- See comment above for rationale for two explicit CHECKPOINTs. CHECKPOINT; CHECKPOINT; @@ -1301,6 +1318,31 @@ SELECT current_setting('fsync') = 'off' t (1 row) +SELECT sum(writes) AS writes, sum(fsyncs) AS fsyncs + FROM pg_stat_get_backend_io(pg_backend_pid()) + WHERE object = 'relation' \gset my_io_sum_shared_after_ +SELECT :my_io_sum_shared_after_writes >= :my_io_sum_shared_before_writes; + ?column? +---------- + t +(1 row) + +SELECT current_setting('fsync') = 'off' + OR (:my_io_sum_shared_after_fsyncs = :my_io_sum_shared_before_fsyncs + AND :my_io_sum_shared_after_fsyncs= 0); + ?column? +---------- + t +(1 row) + +-- Don't return any rows if querying other backend's stats that are excluded +-- from the backend stats collection (like the checkpointer). +SELECT count(1) = 0 FROM pg_stat_get_backend_io(:checkpointer_pid); + ?column? +---------- + t +(1 row) + -- Change the tablespace so that the table is rewritten directly, then SELECT -- from it to cause it to be read back into shared buffers. SELECT sum(reads) AS io_sum_shared_before_reads @@ -1521,6 +1563,8 @@ SELECT pg_stat_have_stats('io', 0, 0); SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS io_stats_pre_reset FROM pg_stat_io \gset +SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS my_io_stats_pre_reset + FROM pg_stat_get_backend_io(pg_backend_pid()) \gset SELECT pg_stat_reset_shared('io'); pg_stat_reset_shared ---------------------- @@ -1535,6 +1579,30 @@ SELECT :io_stats_post_reset < :io_stats_pre_reset; t (1 row) +SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS my_io_stats_post_reset + FROM pg_stat_get_backend_io(pg_backend_pid()) \gset +-- pg_stat_reset_shared() did not reset backend IO stats +SELECT :my_io_stats_pre_reset <= :my_io_stats_post_reset; + ?column? +---------- + t +(1 row) + +-- but pg_stat_reset_backend_stats() does +SELECT pg_stat_reset_backend_stats(pg_backend_pid()); + pg_stat_reset_backend_stats +----------------------------- + +(1 row) + +SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS my_io_stats_post_backend_reset + FROM pg_stat_get_backend_io(pg_backend_pid()) \gset +SELECT :my_io_stats_pre_reset > :my_io_stats_post_backend_reset; + ?column? +---------- + t +(1 row) + -- test BRIN index doesn't block HOT update CREATE TABLE brin_hot ( id integer PRIMARY KEY, diff --git a/src/test/regress/sql/stats.sql b/src/test/regress/sql/stats.sql index 7147cc2f89..9c925005be 100644 --- a/src/test/regress/sql/stats.sql +++ b/src/test/regress/sql/stats.sql @@ -595,7 +595,7 @@ SELECT pg_stat_get_replication_slot(NULL); SELECT pg_stat_get_subscription_stats(NULL); --- Test that the following operations are tracked in pg_stat_io: +-- Test that the following operations are tracked in pg_stat_io and in backend stats: -- - reads of target blocks into shared buffers -- - writes of shared buffers to permanent storage -- - extends of relations using shared buffers @@ -607,20 +607,32 @@ SELECT pg_stat_get_subscription_stats(NULL); -- Create a regular table and insert some data to generate IOCONTEXT_NORMAL -- extends. +SELECT pid AS checkpointer_pid FROM pg_stat_activity + WHERE backend_type = 'checkpointer' \gset SELECT sum(extends) AS io_sum_shared_before_extends FROM pg_stat_io WHERE context = 'normal' AND object = 'relation' \gset +SELECT sum(extends) AS my_io_sum_shared_before_extends + FROM pg_stat_get_backend_io(pg_backend_pid()) + WHERE context = 'normal' AND object = 'relation' \gset SELECT sum(writes) AS writes, sum(fsyncs) AS fsyncs FROM pg_stat_io WHERE object = 'relation' \gset io_sum_shared_before_ +SELECT sum(writes) AS writes, sum(fsyncs) AS fsyncs + FROM pg_stat_get_backend_io(pg_backend_pid()) + WHERE object = 'relation' \gset my_io_sum_shared_before_ CREATE TABLE test_io_shared(a int); INSERT INTO test_io_shared SELECT i FROM generate_series(1,100)i; SELECT pg_stat_force_next_flush(); SELECT sum(extends) AS io_sum_shared_after_extends FROM pg_stat_io WHERE context = 'normal' AND object = 'relation' \gset SELECT :io_sum_shared_after_extends > :io_sum_shared_before_extends; +SELECT sum(extends) AS my_io_sum_shared_after_extends + FROM pg_stat_get_backend_io(pg_backend_pid()) + WHERE context = 'normal' AND object = 'relation' \gset +SELECT :my_io_sum_shared_after_extends > :my_io_sum_shared_before_extends; -- After a checkpoint, there should be some additional IOCONTEXT_NORMAL writes --- and fsyncs. +-- and fsyncs in the global stats (not for the backend). -- See comment above for rationale for two explicit CHECKPOINTs. CHECKPOINT; CHECKPOINT; @@ -630,6 +642,17 @@ SELECT sum(writes) AS writes, sum(fsyncs) AS fsyncs SELECT :io_sum_shared_after_writes > :io_sum_shared_before_writes; SELECT current_setting('fsync') = 'off' OR :io_sum_shared_after_fsyncs > :io_sum_shared_before_fsyncs; +SELECT sum(writes) AS writes, sum(fsyncs) AS fsyncs + FROM pg_stat_get_backend_io(pg_backend_pid()) + WHERE object = 'relation' \gset my_io_sum_shared_after_ +SELECT :my_io_sum_shared_after_writes >= :my_io_sum_shared_before_writes; +SELECT current_setting('fsync') = 'off' + OR (:my_io_sum_shared_after_fsyncs = :my_io_sum_shared_before_fsyncs + AND :my_io_sum_shared_after_fsyncs= 0); + +-- Don't return any rows if querying other backend's stats that are excluded +-- from the backend stats collection (like the checkpointer). +SELECT count(1) = 0 FROM pg_stat_get_backend_io(:checkpointer_pid); -- Change the tablespace so that the table is rewritten directly, then SELECT -- from it to cause it to be read back into shared buffers. @@ -762,10 +785,21 @@ SELECT :io_sum_bulkwrite_strategy_extends_after > :io_sum_bulkwrite_strategy_ext SELECT pg_stat_have_stats('io', 0, 0); SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS io_stats_pre_reset FROM pg_stat_io \gset +SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS my_io_stats_pre_reset + FROM pg_stat_get_backend_io(pg_backend_pid()) \gset SELECT pg_stat_reset_shared('io'); SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS io_stats_post_reset FROM pg_stat_io \gset SELECT :io_stats_post_reset < :io_stats_pre_reset; +SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS my_io_stats_post_reset + FROM pg_stat_get_backend_io(pg_backend_pid()) \gset +-- pg_stat_reset_shared() did not reset backend IO stats +SELECT :my_io_stats_pre_reset <= :my_io_stats_post_reset; +-- but pg_stat_reset_backend_stats() does +SELECT pg_stat_reset_backend_stats(pg_backend_pid()); +SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS my_io_stats_post_backend_reset + FROM pg_stat_get_backend_io(pg_backend_pid()) \gset +SELECT :my_io_stats_pre_reset > :my_io_stats_post_backend_reset; -- test BRIN index doesn't block HOT update diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index ce33e55bf1..398dd92527 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2121,6 +2121,7 @@ PgFdwSamplingMethod PgFdwScanState PgIfAddrCallback PgStatShared_Archiver +PgStatShared_Backend PgStatShared_BgWriter PgStatShared_Checkpointer PgStatShared_Common @@ -2136,6 +2137,8 @@ PgStatShared_SLRU PgStatShared_Subscription PgStatShared_Wal PgStat_ArchiverStats +PgStat_Backend +PgStat_BackendPendingIO PgStat_BackendSubEntry PgStat_BgWriterStats PgStat_BktypeIO -- 2.34.1 --fWBP7ZYU669/m9FI-- ^ permalink raw reply [nested|flat] 25+ messages in thread
* [PATCH v7] per backend I/O statistics @ 2024-10-28 12:50 Bertrand Drouvot <[email protected]> 0 siblings, 0 replies; 25+ messages in thread From: Bertrand Drouvot @ 2024-10-28 12:50 UTC (permalink / raw) While pg_stat_io provides cluster-wide I/O statistics, this commit adds the ability to track and display per backend I/O statistics. It adds a new statistics kind and 2 new functions: - pg_stat_reset_single_backend_io_counters() to be able to reset the I/O stats for a given backend pid. - pg_stat_get_backend_io() to retrieve I/O statistics for a given backend pid. The new KIND is named PGSTAT_KIND_PER_BACKEND as it could be used in the future to store other statistics (than the I/O ones) per backend. The new KIND is a variable-numbered one and has an automatic cap on the maximum number of entries (as its hash key contains the proc number). There is no need to write the per backend I/O stats to disk (no point to see stats for backends that do not exist anymore after a re-start), so using "write_to_file = false". Note that per backend I/O statistics are not collected for the checkpointer, the background writer, the startup process and the autovacuum launcher as those are already visible in pg_stat_io and there is only one of those. XXX: Bump catalog version needs to be done. --- doc/src/sgml/config.sgml | 8 +- doc/src/sgml/monitoring.sgml | 37 ++++ src/backend/catalog/system_functions.sql | 2 + src/backend/utils/activity/Makefile | 1 + src/backend/utils/activity/backend_status.c | 4 + src/backend/utils/activity/meson.build | 1 + src/backend/utils/activity/pgstat.c | 19 +- src/backend/utils/activity/pgstat_backend.c | 182 +++++++++++++++++++ src/backend/utils/activity/pgstat_io.c | 25 ++- src/backend/utils/activity/pgstat_relation.c | 2 + src/backend/utils/adt/pgstatfuncs.c | 166 +++++++++++++++++ src/include/catalog/pg_proc.dat | 14 ++ src/include/pgstat.h | 31 +++- src/include/utils/pgstat_internal.h | 14 ++ src/test/regress/expected/stats.out | 72 +++++++- src/test/regress/sql/stats.sql | 38 +++- src/tools/pgindent/typedefs.list | 3 + 17 files changed, 598 insertions(+), 21 deletions(-) 10.0% doc/src/sgml/ 30.9% src/backend/utils/activity/ 21.8% src/backend/utils/adt/ 4.6% src/include/catalog/ 7.1% src/include/ 13.0% src/test/regress/expected/ 11.6% src/test/regress/sql/ diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index e0c8325a39..8afca9b110 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -8403,9 +8403,11 @@ COPY postgres_log FROM '/full/path/to/logfile.csv' WITH csv; displayed in <link linkend="monitoring-pg-stat-database-view"> <structname>pg_stat_database</structname></link>, <link linkend="monitoring-pg-stat-io-view"> - <structname>pg_stat_io</structname></link>, in the output of - <xref linkend="sql-explain"/> when the <literal>BUFFERS</literal> option - is used, in the output of <xref linkend="sql-vacuum"/> when + <structname>pg_stat_io</structname></link>, in the output of the + <link linkend="pg-stat-get-backend-io"> + <function>pg_stat_get_backend_io()</function></link> function, in the + output of <xref linkend="sql-explain"/> when the <literal>BUFFERS</literal> + option is used, in the output of <xref linkend="sql-vacuum"/> when the <literal>VERBOSE</literal> option is used, by autovacuum for auto-vacuums and auto-analyzes, when <xref linkend="guc-log-autovacuum-min-duration"/> is set and by diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml index 840d7f8161..fb13c462d1 100644 --- a/doc/src/sgml/monitoring.sgml +++ b/doc/src/sgml/monitoring.sgml @@ -4790,6 +4790,25 @@ description | Waiting for a newly initialized WAL file to reach durable storage </para></entry> </row> + <row> + <entry id="pg-stat-get-backend-io" role="func_table_entry"><para role="func_signature"> + <indexterm> + <primary>pg_stat_get_backend_io</primary> + </indexterm> + <function>pg_stat_get_backend_io</function> ( <type>integer</type> ) + <returnvalue>setof record</returnvalue> + </para> + <para> + Returns I/O statistics about the backend with the specified + process ID. The output fields are exactly the same as the ones in the + <link linkend="monitoring-pg-stat-io-view"> <structname>pg_stat_io</structname></link> + view. The function does not return I/O statistics for the checkpointer, + the background writer, the startup process and the autovacuum launcher + as they are already visible in the <link linkend="monitoring-pg-stat-io-view"> <structname>pg_stat_io</structname></link> + view and there is only one of those. + </para></entry> + </row> + <row> <entry role="func_table_entry"><para role="func_signature"> <indexterm> @@ -4971,6 +4990,24 @@ description | Waiting for a newly initialized WAL file to reach durable storage </para></entry> </row> + <row> + <entry role="func_table_entry"><para role="func_signature"> + <indexterm> + <primary>pg_stat_reset_single_backend_io_counters</primary> + </indexterm> + <function>pg_stat_reset_single_backend_io_counters</function> ( <type>integer</type> ) + <returnvalue>void</returnvalue> + </para> + <para> + Resets I/O statistics for a single backend with the specified process ID + to zero. + </para> + <para> + This function is restricted to superusers by default, but other users + can be granted EXECUTE to run the function. + </para></entry> + </row> + <row> <entry role="func_table_entry"><para role="func_signature"> <indexterm> diff --git a/src/backend/catalog/system_functions.sql b/src/backend/catalog/system_functions.sql index c51dfca802..1b21b242cd 100644 --- a/src/backend/catalog/system_functions.sql +++ b/src/backend/catalog/system_functions.sql @@ -711,6 +711,8 @@ REVOKE EXECUTE ON FUNCTION pg_stat_reset_single_table_counters(oid) FROM public; REVOKE EXECUTE ON FUNCTION pg_stat_reset_single_function_counters(oid) FROM public; +REVOKE EXECUTE ON FUNCTION pg_stat_reset_single_backend_io_counters(integer) FROM public; + REVOKE EXECUTE ON FUNCTION pg_stat_reset_replication_slot(text) FROM public; REVOKE EXECUTE ON FUNCTION pg_stat_have_stats(text, oid, int8) FROM public; diff --git a/src/backend/utils/activity/Makefile b/src/backend/utils/activity/Makefile index b9fd66ea17..24b64a2742 100644 --- a/src/backend/utils/activity/Makefile +++ b/src/backend/utils/activity/Makefile @@ -20,6 +20,7 @@ OBJS = \ backend_status.o \ pgstat.o \ pgstat_archiver.o \ + pgstat_backend.o \ pgstat_bgwriter.o \ pgstat_checkpointer.o \ pgstat_database.o \ diff --git a/src/backend/utils/activity/backend_status.c b/src/backend/utils/activity/backend_status.c index 22c6dc378c..29779ff99a 100644 --- a/src/backend/utils/activity/backend_status.c +++ b/src/backend/utils/activity/backend_status.c @@ -249,6 +249,10 @@ pgstat_beinit(void) Assert(MyProcNumber >= 0 && MyProcNumber < NumBackendStatSlots); MyBEEntry = &BackendStatusArray[MyProcNumber]; + /* Create the per-backend statistics entry */ + if (pgstat_tracks_per_backend_bktype(MyBackendType)) + pgstat_create_backend_stat(MyProcNumber); + /* Set up a process-exit hook to clean up */ on_shmem_exit(pgstat_beshutdown_hook, 0); } diff --git a/src/backend/utils/activity/meson.build b/src/backend/utils/activity/meson.build index f73c22905c..380d3dd70c 100644 --- a/src/backend/utils/activity/meson.build +++ b/src/backend/utils/activity/meson.build @@ -5,6 +5,7 @@ backend_sources += files( 'backend_status.c', 'pgstat.c', 'pgstat_archiver.c', + 'pgstat_backend.c', 'pgstat_bgwriter.c', 'pgstat_checkpointer.c', 'pgstat_database.c', diff --git a/src/backend/utils/activity/pgstat.c b/src/backend/utils/activity/pgstat.c index 18b7d9b47d..738ee778bb 100644 --- a/src/backend/utils/activity/pgstat.c +++ b/src/backend/utils/activity/pgstat.c @@ -77,6 +77,7 @@ * * Each statistics kind is handled in a dedicated file: * - pgstat_archiver.c + * - pgstat_backend.c * - pgstat_bgwriter.c * - pgstat_checkpointer.c * - pgstat_database.c @@ -358,6 +359,22 @@ static const PgStat_KindInfo pgstat_kind_builtin_infos[PGSTAT_KIND_BUILTIN_SIZE] .reset_timestamp_cb = pgstat_subscription_reset_timestamp_cb, }, + [PGSTAT_KIND_PER_BACKEND] = { + .name = "per-backend", + + .fixed_amount = false, + .write_to_file = false, + + .accessed_across_databases = true, + + .shared_size = sizeof(PgStatShared_Backend), + .shared_data_off = offsetof(PgStatShared_Backend, stats), + .shared_data_len = sizeof(((PgStatShared_Backend *) 0)->stats), + .pending_size = sizeof(PgStat_BackendPendingIO), + + .flush_pending_cb = pgstat_per_backend_flush_cb, + .reset_timestamp_cb = pgstat_backend_reset_timestamp_cb, + }, /* stats for fixed-numbered (mostly 1) objects */ @@ -768,7 +785,7 @@ pgstat_report_stat(bool force) partial_flush = false; - /* flush database / relation / function / ... stats */ + /* flush database / relation / function / backend / ... stats */ partial_flush |= pgstat_flush_pending_entries(nowait); /* flush of fixed-numbered stats */ diff --git a/src/backend/utils/activity/pgstat_backend.c b/src/backend/utils/activity/pgstat_backend.c new file mode 100644 index 0000000000..dc7bf385c2 --- /dev/null +++ b/src/backend/utils/activity/pgstat_backend.c @@ -0,0 +1,182 @@ +/* ------------------------------------------------------------------------- + * + * pgstat_backend.c + * Implementation of per-backend statistics. + * + * This file contains the implementation of per-backend statistics. It is kept + * separate from pgstat.c to enforce the line between the statistics access / + * storage implementation and the details about individual types of statistics. + * + * Copyright (c) 2024, PostgreSQL Global Development Group + * + * IDENTIFICATION + * src/backend/utils/activity/pgstat_backend.c + * ------------------------------------------------------------------------- + */ + +#include "postgres.h" + +#include "utils/pgstat_internal.h" + +/* + * Returns backend's IO stats. + */ +PgStat_Backend * +pgstat_fetch_proc_stat_io(ProcNumber procNumber) +{ + PgStat_Backend *backend_entry; + + backend_entry = (PgStat_Backend *) pgstat_fetch_entry(PGSTAT_KIND_PER_BACKEND, + InvalidOid, procNumber); + + return backend_entry; +} + +/* + * Flush out locally pending backend statistics + * + * If no stats have been recorded, this function returns false. + */ +bool +pgstat_per_backend_flush_cb(PgStat_EntryRef *entry_ref, bool nowait) +{ + PgStatShared_Backend *shbackendioent; + PgStat_BackendPendingIO *pendingent; + PgStat_BktypeIO *bktype_shstats; + + if (!pgstat_lock_entry(entry_ref, nowait)) + return false; + + shbackendioent = (PgStatShared_Backend *) entry_ref->shared_stats; + bktype_shstats = &shbackendioent->stats.stats; + pendingent = (PgStat_BackendPendingIO *) entry_ref->pending; + + for (int io_object = 0; io_object < IOOBJECT_NUM_TYPES; io_object++) + { + for (int io_context = 0; io_context < IOCONTEXT_NUM_TYPES; io_context++) + { + for (int io_op = 0; io_op < IOOP_NUM_TYPES; io_op++) + { + instr_time time; + + bktype_shstats->counts[io_object][io_context][io_op] += + pendingent->counts[io_object][io_context][io_op]; + + time = pendingent->pending_times[io_object][io_context][io_op]; + + bktype_shstats->times[io_object][io_context][io_op] += + INSTR_TIME_GET_MICROSEC(time); + } + } + } + + pgstat_unlock_entry(entry_ref); + + return true; +} + +/* + * Simpler wrapper of pgstat_per_backend_flush_cb() + */ +void +pgstat_flush_per_backend(bool nowait) +{ + if (pgstat_tracks_per_backend_bktype(MyBackendType)) + { + PgStat_EntryRef *entry_ref; + + entry_ref = pgstat_get_entry_ref(PGSTAT_KIND_PER_BACKEND, InvalidOid, + MyProcNumber, false, NULL); + (void) pgstat_per_backend_flush_cb(entry_ref, nowait); + } +} + +/* + * Create the per-backend statistics entry for procnum. + */ +void +pgstat_create_backend_stat(ProcNumber procnum) +{ + PgStat_EntryRef *entry_ref; + PgStatShared_Backend *shstatent; + + entry_ref = pgstat_prep_pending_entry(PGSTAT_KIND_PER_BACKEND, InvalidOid, + procnum, NULL); + + shstatent = (PgStatShared_Backend *) entry_ref->shared_stats; + + /* + * NB: need to accept that there might be stats from an older backend, + * e.g. if we previously used this proc number. + */ + memset(&shstatent->stats, 0, sizeof(shstatent->stats)); +} + +/* + * Find or create a local PgStat_BackendPendingIO entry for procnum. + */ +PgStat_BackendPendingIO * +pgstat_prep_per_backend_pending(ProcNumber procnum) +{ + PgStat_EntryRef *entry_ref; + + entry_ref = pgstat_prep_pending_entry(PGSTAT_KIND_PER_BACKEND, InvalidOid, + procnum, NULL); + + return entry_ref->pending; +} + +/* + * per-backend statistics are not collected for all BackendTypes. + * + * The following BackendTypes do not participate in the per-backend stats + * subsystem: + * - The same and for the same reasons as in pgstat_tracks_io_bktype(). + * - B_BG_WRITER, B_CHECKPOINTER, B_STARTUP and B_AUTOVAC_LAUNCHER because their + * I/O stats are already visible in pg_stat_io and there is only one of those. + * + * Function returns true if BackendType participates in the per-backend stats + * subsystem for IO and false if it does not. + * + * When adding a new BackendType, also consider adding relevant restrictions to + * pgstat_tracks_io_object() and pgstat_tracks_io_op(). + */ +bool +pgstat_tracks_per_backend_bktype(BackendType bktype) +{ + /* + * List every type so that new backend types trigger a warning about + * needing to adjust this switch. + */ + switch (bktype) + { + case B_INVALID: + case B_AUTOVAC_LAUNCHER: + case B_DEAD_END_BACKEND: + case B_ARCHIVER: + case B_LOGGER: + case B_WAL_RECEIVER: + case B_WAL_WRITER: + case B_WAL_SUMMARIZER: + case B_BG_WRITER: + case B_CHECKPOINTER: + case B_STARTUP: + return false; + + case B_AUTOVAC_WORKER: + case B_BACKEND: + case B_BG_WORKER: + case B_STANDALONE_BACKEND: + case B_SLOTSYNC_WORKER: + case B_WAL_SENDER: + return true; + } + + return false; +} + +void +pgstat_backend_reset_timestamp_cb(PgStatShared_Common *header, TimestampTz ts) +{ + ((PgStatShared_Backend *) header)->stats.stat_reset_timestamp = ts; +} diff --git a/src/backend/utils/activity/pgstat_io.c b/src/backend/utils/activity/pgstat_io.c index f9883af2b3..93d34696b9 100644 --- a/src/backend/utils/activity/pgstat_io.c +++ b/src/backend/utils/activity/pgstat_io.c @@ -20,13 +20,7 @@ #include "storage/bufmgr.h" #include "utils/pgstat_internal.h" - -typedef struct PgStat_PendingIO -{ - PgStat_Counter counts[IOOBJECT_NUM_TYPES][IOCONTEXT_NUM_TYPES][IOOP_NUM_TYPES]; - instr_time pending_times[IOOBJECT_NUM_TYPES][IOCONTEXT_NUM_TYPES][IOOP_NUM_TYPES]; -} PgStat_PendingIO; - +typedef PgStat_BackendPendingIO PgStat_PendingIO; static PgStat_PendingIO PendingIOStats; static bool have_iostats = false; @@ -87,6 +81,14 @@ pgstat_count_io_op_n(IOObject io_object, IOContext io_context, IOOp io_op, uint3 Assert((unsigned int) io_op < IOOP_NUM_TYPES); Assert(pgstat_tracks_io_op(MyBackendType, io_object, io_context, io_op)); + if (pgstat_tracks_per_backend_bktype(MyBackendType)) + { + PgStat_PendingIO *entry_ref; + + entry_ref = pgstat_prep_per_backend_pending(MyProcNumber); + entry_ref->counts[io_object][io_context][io_op] += cnt; + } + PendingIOStats.counts[io_object][io_context][io_op] += cnt; have_iostats = true; @@ -148,6 +150,15 @@ pgstat_count_io_op_time(IOObject io_object, IOContext io_context, IOOp io_op, INSTR_TIME_ADD(PendingIOStats.pending_times[io_object][io_context][io_op], io_time); + + if (pgstat_tracks_per_backend_bktype(MyBackendType)) + { + PgStat_PendingIO *entry_ref; + + entry_ref = pgstat_prep_per_backend_pending(MyProcNumber); + INSTR_TIME_ADD(entry_ref->pending_times[io_object][io_context][io_op], + io_time); + } } pgstat_count_io_op_n(io_object, io_context, io_op, cnt); diff --git a/src/backend/utils/activity/pgstat_relation.c b/src/backend/utils/activity/pgstat_relation.c index faba8b64d2..c083c5bf01 100644 --- a/src/backend/utils/activity/pgstat_relation.c +++ b/src/backend/utils/activity/pgstat_relation.c @@ -264,6 +264,7 @@ pgstat_report_vacuum(Oid tableoid, bool shared, * VACUUM command has processed all tables and committed. */ pgstat_flush_io(false); + pgstat_flush_per_backend(false); } /* @@ -350,6 +351,7 @@ pgstat_report_analyze(Relation rel, /* see pgstat_report_vacuum() */ pgstat_flush_io(false); + pgstat_flush_per_backend(false); } /* diff --git a/src/backend/utils/adt/pgstatfuncs.c b/src/backend/utils/adt/pgstatfuncs.c index cdf37403e9..ce4d22e16c 100644 --- a/src/backend/utils/adt/pgstatfuncs.c +++ b/src/backend/utils/adt/pgstatfuncs.c @@ -1474,6 +1474,151 @@ pg_stat_get_io(PG_FUNCTION_ARGS) return (Datum) 0; } +Datum +pg_stat_get_backend_io(PG_FUNCTION_ARGS) +{ + ReturnSetInfo *rsinfo; + PgStat_Backend *backend_stats; + Datum bktype_desc; + PgStat_BktypeIO *bktype_stats; + BackendType bktype; + Datum reset_time; + int num_backends = pgstat_fetch_stat_numbackends(); + int curr_backend; + int pid; + PGPROC *proc; + ProcNumber procNumber; + + InitMaterializedSRF(fcinfo, 0); + rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; + + pid = PG_GETARG_INT32(0); + proc = BackendPidGetProc(pid); + + /* + * Could be an auxiliary process but would not report any stats due to + * pgstat_tracks_per_backend_bktype() anyway. So don't need an extra call + * to AuxiliaryPidGetProc(). + */ + if (!proc) + return (Datum) 0; + + procNumber = GetNumberFromPGProc(proc); + backend_stats = pgstat_fetch_proc_stat_io(procNumber); + + if (!backend_stats) + return (Datum) 0; + + /* just to keep compiler quiet */ + bktype = B_INVALID; + + /* Look for the backend type */ + for (curr_backend = 1; curr_backend <= num_backends; curr_backend++) + { + LocalPgBackendStatus *local_beentry; + PgBackendStatus *beentry; + + /* Get the next one in the list */ + local_beentry = pgstat_get_local_beentry_by_index(curr_backend); + beentry = &local_beentry->backendStatus; + + /* looking for specific PID, ignore all the others */ + if (beentry->st_procpid != pid) + continue; + + bktype = beentry->st_backendType; + break; + } + + bktype_desc = CStringGetTextDatum(GetBackendTypeDesc(bktype)); + bktype_stats = &backend_stats->stats; + reset_time = TimestampTzGetDatum(backend_stats->stat_reset_timestamp); + + /* + * In Assert builds, we can afford an extra loop through all of the + * counters checking that only expected stats are non-zero, since it keeps + * the non-Assert code cleaner. + */ + Assert(pgstat_bktype_io_stats_valid(bktype_stats, bktype)); + + for (int io_obj = 0; io_obj < IOOBJECT_NUM_TYPES; io_obj++) + { + const char *obj_name = pgstat_get_io_object_name(io_obj); + + for (int io_context = 0; io_context < IOCONTEXT_NUM_TYPES; io_context++) + { + const char *context_name = pgstat_get_io_context_name(io_context); + + Datum values[IO_NUM_COLUMNS] = {0}; + bool nulls[IO_NUM_COLUMNS] = {0}; + + /* + * Some combinations of BackendType, IOObject, and IOContext are + * not valid for any type of IOOp. In such cases, omit the entire + * row from the view. + */ + if (!pgstat_tracks_io_object(bktype, io_obj, io_context)) + continue; + + values[IO_COL_BACKEND_TYPE] = bktype_desc; + values[IO_COL_CONTEXT] = CStringGetTextDatum(context_name); + values[IO_COL_OBJECT] = CStringGetTextDatum(obj_name); + if (backend_stats->stat_reset_timestamp != 0) + values[IO_COL_RESET_TIME] = reset_time; + else + nulls[IO_COL_RESET_TIME] = true; + + /* + * Hard-code this to the value of BLCKSZ for now. Future values + * could include XLOG_BLCKSZ, once WAL IO is tracked, and constant + * multipliers, once non-block-oriented IO (e.g. temporary file + * IO) is tracked. + */ + values[IO_COL_CONVERSION] = Int64GetDatum(BLCKSZ); + + for (int io_op = 0; io_op < IOOP_NUM_TYPES; io_op++) + { + int op_idx = pgstat_get_io_op_index(io_op); + int time_idx = pgstat_get_io_time_index(io_op); + + /* + * Some combinations of BackendType and IOOp, of IOContext and + * IOOp, and of IOObject and IOOp are not tracked. Set these + * cells in the view NULL. + */ + if (pgstat_tracks_io_op(bktype, io_obj, io_context, io_op)) + { + PgStat_Counter count = + bktype_stats->counts[io_obj][io_context][io_op]; + + values[op_idx] = Int64GetDatum(count); + } + else + nulls[op_idx] = true; + + /* not every operation is timed */ + if (time_idx == IO_COL_INVALID) + continue; + + if (!nulls[op_idx]) + { + PgStat_Counter time = + bktype_stats->times[io_obj][io_context][io_op]; + + values[time_idx] = Float8GetDatum(pg_stat_us_to_ms(time)); + } + else + nulls[time_idx] = true; + } + + tuplestore_putvalues(rsinfo->setResult, rsinfo->setDesc, + values, nulls); + } + } + + return (Datum) 0; +} + /* * Returns statistics of WAL activity */ @@ -1779,6 +1924,27 @@ pg_stat_reset_single_function_counters(PG_FUNCTION_ARGS) PG_RETURN_VOID(); } +Datum +pg_stat_reset_single_backend_io_counters(PG_FUNCTION_ARGS) +{ + PGPROC *proc; + int backend_pid = PG_GETARG_INT32(0); + + proc = BackendPidGetProc(backend_pid); + + /* + * Could be an auxiliary process but would not report any stats due to + * pgstat_tracks_per_backend_bktype() anyway. So don't need an extra call + * to AuxiliaryPidGetProc(). + */ + if (!proc) + PG_RETURN_VOID(); + + pgstat_reset(PGSTAT_KIND_PER_BACKEND, InvalidOid, GetNumberFromPGProc(proc)); + + PG_RETURN_VOID(); +} + /* Reset SLRU counters (a specific one or all of them). */ Datum pg_stat_reset_slru(PG_FUNCTION_ARGS) diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index 0f22c21723..55dec76795 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -5913,6 +5913,15 @@ proargnames => '{backend_type,object,context,reads,read_time,writes,write_time,writebacks,writeback_time,extends,extend_time,op_bytes,hits,evictions,reuses,fsyncs,fsync_time,stats_reset}', prosrc => 'pg_stat_get_io' }, +{ oid => '8806', descr => 'statistics: per backend IO statistics', + proname => 'pg_stat_get_backend_io', prorows => '5', proretset => 't', + provolatile => 'v', proparallel => 'r', prorettype => 'record', + proargtypes => 'int4', + proallargtypes => '{int4,text,text,text,int8,float8,int8,float8,int8,float8,int8,float8,int8,int8,int8,int8,int8,float8,timestamptz}', + proargmodes => '{i,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o}', + proargnames => '{backend_pid,backend_type,object,context,reads,read_time,writes,write_time,writebacks,writeback_time,extends,extend_time,op_bytes,hits,evictions,reuses,fsyncs,fsync_time,stats_reset}', + prosrc => 'pg_stat_get_backend_io' }, + { oid => '1136', descr => 'statistics: information about WAL activity', proname => 'pg_stat_get_wal', proisstrict => 'f', provolatile => 's', proparallel => 'r', prorettype => 'record', proargtypes => '', @@ -6052,6 +6061,11 @@ proname => 'pg_stat_reset_single_function_counters', provolatile => 'v', prorettype => 'void', proargtypes => 'oid', prosrc => 'pg_stat_reset_single_function_counters' }, +{ oid => '9987', + descr => 'statistics: reset collected IO statistics for a single backend', + proname => 'pg_stat_reset_single_backend_io_counters', provolatile => 'v', + prorettype => 'void', proargtypes => 'int4', + prosrc => 'pg_stat_reset_single_backend_io_counters' }, { oid => '2307', descr => 'statistics: reset collected statistics for a single SLRU', proname => 'pg_stat_reset_slru', proisstrict => 'f', provolatile => 'v', diff --git a/src/include/pgstat.h b/src/include/pgstat.h index 59c28b4aca..b7e8dcb1fd 100644 --- a/src/include/pgstat.h +++ b/src/include/pgstat.h @@ -49,14 +49,15 @@ #define PGSTAT_KIND_FUNCTION 3 /* per-function statistics */ #define PGSTAT_KIND_REPLSLOT 4 /* per-slot statistics */ #define PGSTAT_KIND_SUBSCRIPTION 5 /* per-subscription statistics */ +#define PGSTAT_KIND_PER_BACKEND 6 /* per-backend statistics */ /* stats for fixed-numbered objects */ -#define PGSTAT_KIND_ARCHIVER 6 -#define PGSTAT_KIND_BGWRITER 7 -#define PGSTAT_KIND_CHECKPOINTER 8 -#define PGSTAT_KIND_IO 9 -#define PGSTAT_KIND_SLRU 10 -#define PGSTAT_KIND_WAL 11 +#define PGSTAT_KIND_ARCHIVER 7 +#define PGSTAT_KIND_BGWRITER 8 +#define PGSTAT_KIND_CHECKPOINTER 9 +#define PGSTAT_KIND_IO 10 +#define PGSTAT_KIND_SLRU 11 +#define PGSTAT_KIND_WAL 12 #define PGSTAT_KIND_BUILTIN_MIN PGSTAT_KIND_DATABASE #define PGSTAT_KIND_BUILTIN_MAX PGSTAT_KIND_WAL @@ -347,12 +348,23 @@ typedef struct PgStat_BktypeIO PgStat_Counter times[IOOBJECT_NUM_TYPES][IOCONTEXT_NUM_TYPES][IOOP_NUM_TYPES]; } PgStat_BktypeIO; +typedef struct PgStat_BackendPendingIO +{ + PgStat_Counter counts[IOOBJECT_NUM_TYPES][IOCONTEXT_NUM_TYPES][IOOP_NUM_TYPES]; + instr_time pending_times[IOOBJECT_NUM_TYPES][IOCONTEXT_NUM_TYPES][IOOP_NUM_TYPES]; +} PgStat_BackendPendingIO; + typedef struct PgStat_IO { TimestampTz stat_reset_timestamp; PgStat_BktypeIO stats[BACKEND_NUM_TYPES]; } PgStat_IO; +typedef struct PgStat_Backend +{ + TimestampTz stat_reset_timestamp; + PgStat_BktypeIO stats; +} PgStat_Backend; typedef struct PgStat_StatDBEntry { @@ -534,6 +546,13 @@ extern bool pgstat_have_entry(PgStat_Kind kind, Oid dboid, uint64 objid); extern void pgstat_report_archiver(const char *xlog, bool failed); extern PgStat_ArchiverStats *pgstat_fetch_stat_archiver(void); +/* + * Functions in pgstat_backend.c + */ + +extern PgStat_Backend *pgstat_fetch_proc_stat_io(ProcNumber procNumber); +extern bool pgstat_tracks_per_backend_bktype(BackendType bktype); +extern void pgstat_create_backend_stat(ProcNumber procnum); /* * Functions in pgstat_bgwriter.c diff --git a/src/include/utils/pgstat_internal.h b/src/include/utils/pgstat_internal.h index 7338bc1e28..141e16d1d3 100644 --- a/src/include/utils/pgstat_internal.h +++ b/src/include/utils/pgstat_internal.h @@ -450,6 +450,11 @@ typedef struct PgStatShared_ReplSlot PgStat_StatReplSlotEntry stats; } PgStatShared_ReplSlot; +typedef struct PgStatShared_Backend +{ + PgStatShared_Common header; + PgStat_Backend stats; +} PgStatShared_Backend; /* * Central shared memory entry for the cumulative stats system. @@ -604,6 +609,15 @@ extern void pgstat_archiver_init_shmem_cb(void *stats); extern void pgstat_archiver_reset_all_cb(TimestampTz ts); extern void pgstat_archiver_snapshot_cb(void); +/* + * Functions in pgstat_backend.c + */ + +extern void pgstat_flush_per_backend(bool nowait); + +extern PgStat_BackendPendingIO *pgstat_prep_per_backend_pending(ProcNumber procnum); +extern bool pgstat_per_backend_flush_cb(PgStat_EntryRef *entry_ref, bool nowait); +extern void pgstat_backend_reset_timestamp_cb(PgStatShared_Common *header, TimestampTz ts); /* * Functions in pgstat_bgwriter.c diff --git a/src/test/regress/expected/stats.out b/src/test/regress/expected/stats.out index 56771f83ed..ba8c3cfc88 100644 --- a/src/test/regress/expected/stats.out +++ b/src/test/regress/expected/stats.out @@ -1249,7 +1249,7 @@ SELECT pg_stat_get_subscription_stats(NULL); (1 row) --- Test that the following operations are tracked in pg_stat_io: +-- Test that the following operations are tracked in pg_stat_io and in per-backend stats: -- - reads of target blocks into shared buffers -- - writes of shared buffers to permanent storage -- - extends of relations using shared buffers @@ -1259,11 +1259,19 @@ SELECT pg_stat_get_subscription_stats(NULL); -- be sure of the state of shared buffers at the point the test is run. -- Create a regular table and insert some data to generate IOCONTEXT_NORMAL -- extends. +SELECT pid AS checkpointer_pid FROM pg_stat_activity + WHERE backend_type = 'checkpointer' \gset SELECT sum(extends) AS io_sum_shared_before_extends FROM pg_stat_io WHERE context = 'normal' AND object = 'relation' \gset +SELECT sum(extends) AS my_io_sum_shared_before_extends + FROM pg_stat_get_backend_io(pg_backend_pid()) + WHERE context = 'normal' AND object = 'relation' \gset SELECT sum(writes) AS writes, sum(fsyncs) AS fsyncs FROM pg_stat_io WHERE object = 'relation' \gset io_sum_shared_before_ +SELECT sum(writes) AS writes, sum(fsyncs) AS fsyncs + FROM pg_stat_get_backend_io(pg_backend_pid()) + WHERE object = 'relation' \gset my_io_sum_shared_before_ CREATE TABLE test_io_shared(a int); INSERT INTO test_io_shared SELECT i FROM generate_series(1,100)i; SELECT pg_stat_force_next_flush(); @@ -1280,8 +1288,17 @@ SELECT :io_sum_shared_after_extends > :io_sum_shared_before_extends; t (1 row) +SELECT sum(extends) AS my_io_sum_shared_after_extends + FROM pg_stat_get_backend_io(pg_backend_pid()) + WHERE context = 'normal' AND object = 'relation' \gset +SELECT :my_io_sum_shared_after_extends > :my_io_sum_shared_before_extends; + ?column? +---------- + t +(1 row) + -- After a checkpoint, there should be some additional IOCONTEXT_NORMAL writes --- and fsyncs. +-- and fsyncs in the global stats (not for the backend). -- See comment above for rationale for two explicit CHECKPOINTs. CHECKPOINT; CHECKPOINT; @@ -1301,6 +1318,31 @@ SELECT current_setting('fsync') = 'off' t (1 row) +SELECT sum(writes) AS writes, sum(fsyncs) AS fsyncs + FROM pg_stat_get_backend_io(pg_backend_pid()) + WHERE object = 'relation' \gset my_io_sum_shared_after_ +SELECT :my_io_sum_shared_after_writes >= :my_io_sum_shared_before_writes; + ?column? +---------- + t +(1 row) + +SELECT current_setting('fsync') = 'off' + OR (:my_io_sum_shared_after_fsyncs = :my_io_sum_shared_before_fsyncs + AND :my_io_sum_shared_after_fsyncs= 0); + ?column? +---------- + t +(1 row) + +-- Don't return any rows if querying other backend's stats that are excluded +-- from the per-backend stats collection (like the checkpointer). +SELECT count(1) = 0 FROM pg_stat_get_backend_io(:checkpointer_pid); + ?column? +---------- + t +(1 row) + -- Change the tablespace so that the table is rewritten directly, then SELECT -- from it to cause it to be read back into shared buffers. SELECT sum(reads) AS io_sum_shared_before_reads @@ -1521,6 +1563,8 @@ SELECT pg_stat_have_stats('io', 0, 0); SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS io_stats_pre_reset FROM pg_stat_io \gset +SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS my_io_stats_pre_reset + FROM pg_stat_get_backend_io(pg_backend_pid()) \gset SELECT pg_stat_reset_shared('io'); pg_stat_reset_shared ---------------------- @@ -1535,6 +1579,30 @@ SELECT :io_stats_post_reset < :io_stats_pre_reset; t (1 row) +SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS my_io_stats_post_reset + FROM pg_stat_get_backend_io(pg_backend_pid()) \gset +-- pg_stat_reset_shared() did not reset backend IO stats +SELECT :my_io_stats_pre_reset <= :my_io_stats_post_reset; + ?column? +---------- + t +(1 row) + +-- but pg_stat_reset_single_backend_io_counters() does +SELECT pg_stat_reset_single_backend_io_counters(pg_backend_pid()); + pg_stat_reset_single_backend_io_counters +------------------------------------------ + +(1 row) + +SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS my_io_stats_post_backend_reset + FROM pg_stat_get_backend_io(pg_backend_pid()) \gset +SELECT :my_io_stats_pre_reset > :my_io_stats_post_backend_reset; + ?column? +---------- + t +(1 row) + -- test BRIN index doesn't block HOT update CREATE TABLE brin_hot ( id integer PRIMARY KEY, diff --git a/src/test/regress/sql/stats.sql b/src/test/regress/sql/stats.sql index 7147cc2f89..f17799402f 100644 --- a/src/test/regress/sql/stats.sql +++ b/src/test/regress/sql/stats.sql @@ -595,7 +595,7 @@ SELECT pg_stat_get_replication_slot(NULL); SELECT pg_stat_get_subscription_stats(NULL); --- Test that the following operations are tracked in pg_stat_io: +-- Test that the following operations are tracked in pg_stat_io and in per-backend stats: -- - reads of target blocks into shared buffers -- - writes of shared buffers to permanent storage -- - extends of relations using shared buffers @@ -607,20 +607,32 @@ SELECT pg_stat_get_subscription_stats(NULL); -- Create a regular table and insert some data to generate IOCONTEXT_NORMAL -- extends. +SELECT pid AS checkpointer_pid FROM pg_stat_activity + WHERE backend_type = 'checkpointer' \gset SELECT sum(extends) AS io_sum_shared_before_extends FROM pg_stat_io WHERE context = 'normal' AND object = 'relation' \gset +SELECT sum(extends) AS my_io_sum_shared_before_extends + FROM pg_stat_get_backend_io(pg_backend_pid()) + WHERE context = 'normal' AND object = 'relation' \gset SELECT sum(writes) AS writes, sum(fsyncs) AS fsyncs FROM pg_stat_io WHERE object = 'relation' \gset io_sum_shared_before_ +SELECT sum(writes) AS writes, sum(fsyncs) AS fsyncs + FROM pg_stat_get_backend_io(pg_backend_pid()) + WHERE object = 'relation' \gset my_io_sum_shared_before_ CREATE TABLE test_io_shared(a int); INSERT INTO test_io_shared SELECT i FROM generate_series(1,100)i; SELECT pg_stat_force_next_flush(); SELECT sum(extends) AS io_sum_shared_after_extends FROM pg_stat_io WHERE context = 'normal' AND object = 'relation' \gset SELECT :io_sum_shared_after_extends > :io_sum_shared_before_extends; +SELECT sum(extends) AS my_io_sum_shared_after_extends + FROM pg_stat_get_backend_io(pg_backend_pid()) + WHERE context = 'normal' AND object = 'relation' \gset +SELECT :my_io_sum_shared_after_extends > :my_io_sum_shared_before_extends; -- After a checkpoint, there should be some additional IOCONTEXT_NORMAL writes --- and fsyncs. +-- and fsyncs in the global stats (not for the backend). -- See comment above for rationale for two explicit CHECKPOINTs. CHECKPOINT; CHECKPOINT; @@ -630,6 +642,17 @@ SELECT sum(writes) AS writes, sum(fsyncs) AS fsyncs SELECT :io_sum_shared_after_writes > :io_sum_shared_before_writes; SELECT current_setting('fsync') = 'off' OR :io_sum_shared_after_fsyncs > :io_sum_shared_before_fsyncs; +SELECT sum(writes) AS writes, sum(fsyncs) AS fsyncs + FROM pg_stat_get_backend_io(pg_backend_pid()) + WHERE object = 'relation' \gset my_io_sum_shared_after_ +SELECT :my_io_sum_shared_after_writes >= :my_io_sum_shared_before_writes; +SELECT current_setting('fsync') = 'off' + OR (:my_io_sum_shared_after_fsyncs = :my_io_sum_shared_before_fsyncs + AND :my_io_sum_shared_after_fsyncs= 0); + +-- Don't return any rows if querying other backend's stats that are excluded +-- from the per-backend stats collection (like the checkpointer). +SELECT count(1) = 0 FROM pg_stat_get_backend_io(:checkpointer_pid); -- Change the tablespace so that the table is rewritten directly, then SELECT -- from it to cause it to be read back into shared buffers. @@ -762,10 +785,21 @@ SELECT :io_sum_bulkwrite_strategy_extends_after > :io_sum_bulkwrite_strategy_ext SELECT pg_stat_have_stats('io', 0, 0); SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS io_stats_pre_reset FROM pg_stat_io \gset +SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS my_io_stats_pre_reset + FROM pg_stat_get_backend_io(pg_backend_pid()) \gset SELECT pg_stat_reset_shared('io'); SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS io_stats_post_reset FROM pg_stat_io \gset SELECT :io_stats_post_reset < :io_stats_pre_reset; +SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS my_io_stats_post_reset + FROM pg_stat_get_backend_io(pg_backend_pid()) \gset +-- pg_stat_reset_shared() did not reset backend IO stats +SELECT :my_io_stats_pre_reset <= :my_io_stats_post_reset; +-- but pg_stat_reset_single_backend_io_counters() does +SELECT pg_stat_reset_single_backend_io_counters(pg_backend_pid()); +SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS my_io_stats_post_backend_reset + FROM pg_stat_get_backend_io(pg_backend_pid()) \gset +SELECT :my_io_stats_pre_reset > :my_io_stats_post_backend_reset; -- test BRIN index doesn't block HOT update diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index ce33e55bf1..398dd92527 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2121,6 +2121,7 @@ PgFdwSamplingMethod PgFdwScanState PgIfAddrCallback PgStatShared_Archiver +PgStatShared_Backend PgStatShared_BgWriter PgStatShared_Checkpointer PgStatShared_Common @@ -2136,6 +2137,8 @@ PgStatShared_SLRU PgStatShared_Subscription PgStatShared_Wal PgStat_ArchiverStats +PgStat_Backend +PgStat_BackendPendingIO PgStat_BackendSubEntry PgStat_BgWriterStats PgStat_BktypeIO -- 2.34.1 --PnfPtg2CPUf6YsJo-- ^ permalink raw reply [nested|flat] 25+ messages in thread
* [PATCH v8] per backend I/O statistics @ 2024-10-28 12:50 Bertrand Drouvot <[email protected]> 0 siblings, 0 replies; 25+ messages in thread From: Bertrand Drouvot @ 2024-10-28 12:50 UTC (permalink / raw) While pg_stat_io provides cluster-wide I/O statistics, this commit adds the ability to track and display per backend I/O statistics. It adds a new statistics kind and 2 new functions: - pg_stat_reset_backend_stats() to be able to reset the stats for a given backend pid. - pg_stat_get_backend_io() to retrieve I/O statistics for a given backend pid. The new KIND is named PGSTAT_KIND_BACKEND as it could be used in the future to store other statistics (than the I/O ones) per backend. The new KIND is a variable-numbered one and has an automatic cap on the maximum number of entries (as its hash key contains the proc number). There is no need to write the per backend I/O stats to disk (no point to see stats for backends that do not exist anymore after a re-start), so using "write_to_file = false". Note that per backend I/O statistics are not collected for the checkpointer, the background writer, the startup process and the autovacuum launcher as those are already visible in pg_stat_io and there is only one of those. XXX: Bump catalog version needs to be done. --- doc/src/sgml/config.sgml | 8 +- doc/src/sgml/monitoring.sgml | 37 ++++ src/backend/catalog/system_functions.sql | 2 + src/backend/utils/activity/Makefile | 1 + src/backend/utils/activity/backend_status.c | 4 + src/backend/utils/activity/meson.build | 1 + src/backend/utils/activity/pgstat.c | 22 ++- src/backend/utils/activity/pgstat_backend.c | 182 +++++++++++++++++++ src/backend/utils/activity/pgstat_io.c | 25 ++- src/backend/utils/activity/pgstat_relation.c | 2 + src/backend/utils/adt/pgstatfuncs.c | 169 +++++++++++++++++ src/include/catalog/pg_proc.dat | 14 ++ src/include/pgstat.h | 31 +++- src/include/utils/pgstat_internal.h | 14 ++ src/test/regress/expected/stats.out | 72 +++++++- src/test/regress/sql/stats.sql | 38 +++- src/tools/pgindent/typedefs.list | 3 + 17 files changed, 604 insertions(+), 21 deletions(-) 9.9% doc/src/sgml/ 31.2% src/backend/utils/activity/ 22.1% src/backend/utils/adt/ 4.4% src/include/catalog/ 7.1% src/include/ 12.9% src/test/regress/expected/ 11.5% src/test/regress/sql/ diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index e0c8325a39..8afca9b110 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -8403,9 +8403,11 @@ COPY postgres_log FROM '/full/path/to/logfile.csv' WITH csv; displayed in <link linkend="monitoring-pg-stat-database-view"> <structname>pg_stat_database</structname></link>, <link linkend="monitoring-pg-stat-io-view"> - <structname>pg_stat_io</structname></link>, in the output of - <xref linkend="sql-explain"/> when the <literal>BUFFERS</literal> option - is used, in the output of <xref linkend="sql-vacuum"/> when + <structname>pg_stat_io</structname></link>, in the output of the + <link linkend="pg-stat-get-backend-io"> + <function>pg_stat_get_backend_io()</function></link> function, in the + output of <xref linkend="sql-explain"/> when the <literal>BUFFERS</literal> + option is used, in the output of <xref linkend="sql-vacuum"/> when the <literal>VERBOSE</literal> option is used, by autovacuum for auto-vacuums and auto-analyzes, when <xref linkend="guc-log-autovacuum-min-duration"/> is set and by diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml index 840d7f8161..4a9464da61 100644 --- a/doc/src/sgml/monitoring.sgml +++ b/doc/src/sgml/monitoring.sgml @@ -4790,6 +4790,25 @@ description | Waiting for a newly initialized WAL file to reach durable storage </para></entry> </row> + <row> + <entry id="pg-stat-get-backend-io" role="func_table_entry"><para role="func_signature"> + <indexterm> + <primary>pg_stat_get_backend_io</primary> + </indexterm> + <function>pg_stat_get_backend_io</function> ( <type>integer</type> ) + <returnvalue>setof record</returnvalue> + </para> + <para> + Returns I/O statistics about the backend with the specified + process ID. The output fields are exactly the same as the ones in the + <link linkend="monitoring-pg-stat-io-view"> <structname>pg_stat_io</structname></link> + view. The function does not return I/O statistics for the checkpointer, + the background writer, the startup process and the autovacuum launcher + as they are already visible in the <link linkend="monitoring-pg-stat-io-view"> <structname>pg_stat_io</structname></link> + view and there is only one of those. + </para></entry> + </row> + <row> <entry role="func_table_entry"><para role="func_signature"> <indexterm> @@ -4971,6 +4990,24 @@ description | Waiting for a newly initialized WAL file to reach durable storage </para></entry> </row> + <row> + <entry role="func_table_entry"><para role="func_signature"> + <indexterm> + <primary>pg_stat_reset_backend_stats</primary> + </indexterm> + <function>pg_stat_reset_backend_stats</function> ( <type>integer</type> ) + <returnvalue>void</returnvalue> + </para> + <para> + Resets statistics for a single backend with the specified process ID + to zero. + </para> + <para> + This function is restricted to superusers by default, but other users + can be granted EXECUTE to run the function. + </para></entry> + </row> + <row> <entry role="func_table_entry"><para role="func_signature"> <indexterm> diff --git a/src/backend/catalog/system_functions.sql b/src/backend/catalog/system_functions.sql index c51dfca802..14eb99cd47 100644 --- a/src/backend/catalog/system_functions.sql +++ b/src/backend/catalog/system_functions.sql @@ -711,6 +711,8 @@ REVOKE EXECUTE ON FUNCTION pg_stat_reset_single_table_counters(oid) FROM public; REVOKE EXECUTE ON FUNCTION pg_stat_reset_single_function_counters(oid) FROM public; +REVOKE EXECUTE ON FUNCTION pg_stat_reset_backend_stats(integer) FROM public; + REVOKE EXECUTE ON FUNCTION pg_stat_reset_replication_slot(text) FROM public; REVOKE EXECUTE ON FUNCTION pg_stat_have_stats(text, oid, int8) FROM public; diff --git a/src/backend/utils/activity/Makefile b/src/backend/utils/activity/Makefile index b9fd66ea17..24b64a2742 100644 --- a/src/backend/utils/activity/Makefile +++ b/src/backend/utils/activity/Makefile @@ -20,6 +20,7 @@ OBJS = \ backend_status.o \ pgstat.o \ pgstat_archiver.o \ + pgstat_backend.o \ pgstat_bgwriter.o \ pgstat_checkpointer.o \ pgstat_database.o \ diff --git a/src/backend/utils/activity/backend_status.c b/src/backend/utils/activity/backend_status.c index 22c6dc378c..7f74011a00 100644 --- a/src/backend/utils/activity/backend_status.c +++ b/src/backend/utils/activity/backend_status.c @@ -249,6 +249,10 @@ pgstat_beinit(void) Assert(MyProcNumber >= 0 && MyProcNumber < NumBackendStatSlots); MyBEEntry = &BackendStatusArray[MyProcNumber]; + /* Create the backend statistics entry */ + if (pgstat_tracks_backend_bktype(MyBackendType)) + pgstat_create_backend_stat(MyProcNumber); + /* Set up a process-exit hook to clean up */ on_shmem_exit(pgstat_beshutdown_hook, 0); } diff --git a/src/backend/utils/activity/meson.build b/src/backend/utils/activity/meson.build index f73c22905c..380d3dd70c 100644 --- a/src/backend/utils/activity/meson.build +++ b/src/backend/utils/activity/meson.build @@ -5,6 +5,7 @@ backend_sources += files( 'backend_status.c', 'pgstat.c', 'pgstat_archiver.c', + 'pgstat_backend.c', 'pgstat_bgwriter.c', 'pgstat_checkpointer.c', 'pgstat_database.c', diff --git a/src/backend/utils/activity/pgstat.c b/src/backend/utils/activity/pgstat.c index 18b7d9b47d..563f2443f8 100644 --- a/src/backend/utils/activity/pgstat.c +++ b/src/backend/utils/activity/pgstat.c @@ -77,6 +77,7 @@ * * Each statistics kind is handled in a dedicated file: * - pgstat_archiver.c + * - pgstat_backend.c * - pgstat_bgwriter.c * - pgstat_checkpointer.c * - pgstat_database.c @@ -358,6 +359,22 @@ static const PgStat_KindInfo pgstat_kind_builtin_infos[PGSTAT_KIND_BUILTIN_SIZE] .reset_timestamp_cb = pgstat_subscription_reset_timestamp_cb, }, + [PGSTAT_KIND_BACKEND] = { + .name = "backend", + + .fixed_amount = false, + .write_to_file = false, + + .accessed_across_databases = true, + + .shared_size = sizeof(PgStatShared_Backend), + .shared_data_off = offsetof(PgStatShared_Backend, stats), + .shared_data_len = sizeof(((PgStatShared_Backend *) 0)->stats), + .pending_size = sizeof(PgStat_BackendPendingIO), + + .flush_pending_cb = pgstat_backend_flush_cb, + .reset_timestamp_cb = pgstat_backend_reset_timestamp_cb, + }, /* stats for fixed-numbered (mostly 1) objects */ @@ -602,6 +619,9 @@ pgstat_shutdown_hook(int code, Datum arg) Assert(dlist_is_empty(&pgStatPending)); dlist_init(&pgStatPending); + /* drop the backend stats entry */ + pgstat_drop_entry(PGSTAT_KIND_BACKEND, InvalidOid, MyProcNumber); + pgstat_detach_shmem(); #ifdef USE_ASSERT_CHECKING @@ -768,7 +788,7 @@ pgstat_report_stat(bool force) partial_flush = false; - /* flush database / relation / function / ... stats */ + /* flush database / relation / function / backend / ... stats */ partial_flush |= pgstat_flush_pending_entries(nowait); /* flush of fixed-numbered stats */ diff --git a/src/backend/utils/activity/pgstat_backend.c b/src/backend/utils/activity/pgstat_backend.c new file mode 100644 index 0000000000..e2d83024c2 --- /dev/null +++ b/src/backend/utils/activity/pgstat_backend.c @@ -0,0 +1,182 @@ +/* ------------------------------------------------------------------------- + * + * pgstat_backend.c + * Implementation of backend statistics. + * + * This file contains the implementation of backend statistics. It is kept + * separate from pgstat.c to enforce the line between the statistics access / + * storage implementation and the details about individual types of statistics. + * + * Copyright (c) 2024, PostgreSQL Global Development Group + * + * IDENTIFICATION + * src/backend/utils/activity/pgstat_backend.c + * ------------------------------------------------------------------------- + */ + +#include "postgres.h" + +#include "utils/pgstat_internal.h" + +/* + * Returns backend's IO stats. + */ +PgStat_Backend * +pgstat_fetch_proc_stat_io(ProcNumber procNumber) +{ + PgStat_Backend *backend_entry; + + backend_entry = (PgStat_Backend *) pgstat_fetch_entry(PGSTAT_KIND_BACKEND, + InvalidOid, procNumber); + + return backend_entry; +} + +/* + * Flush out locally pending backend statistics + * + * If no stats have been recorded, this function returns false. + */ +bool +pgstat_backend_flush_cb(PgStat_EntryRef *entry_ref, bool nowait) +{ + PgStatShared_Backend *shbackendioent; + PgStat_BackendPendingIO *pendingent; + PgStat_BktypeIO *bktype_shstats; + + if (!pgstat_lock_entry(entry_ref, nowait)) + return false; + + shbackendioent = (PgStatShared_Backend *) entry_ref->shared_stats; + bktype_shstats = &shbackendioent->stats.stats; + pendingent = (PgStat_BackendPendingIO *) entry_ref->pending; + + for (int io_object = 0; io_object < IOOBJECT_NUM_TYPES; io_object++) + { + for (int io_context = 0; io_context < IOCONTEXT_NUM_TYPES; io_context++) + { + for (int io_op = 0; io_op < IOOP_NUM_TYPES; io_op++) + { + instr_time time; + + bktype_shstats->counts[io_object][io_context][io_op] += + pendingent->counts[io_object][io_context][io_op]; + + time = pendingent->pending_times[io_object][io_context][io_op]; + + bktype_shstats->times[io_object][io_context][io_op] += + INSTR_TIME_GET_MICROSEC(time); + } + } + } + + pgstat_unlock_entry(entry_ref); + + return true; +} + +/* + * Simpler wrapper of pgstat_backend_flush_cb() + */ +void +pgstat_flush_backend(bool nowait) +{ + if (pgstat_tracks_backend_bktype(MyBackendType)) + { + PgStat_EntryRef *entry_ref; + + entry_ref = pgstat_get_entry_ref(PGSTAT_KIND_BACKEND, InvalidOid, + MyProcNumber, false, NULL); + (void) pgstat_backend_flush_cb(entry_ref, nowait); + } +} + +/* + * Create the backend statistics entry for procnum. + */ +void +pgstat_create_backend_stat(ProcNumber procnum) +{ + PgStat_EntryRef *entry_ref; + PgStatShared_Backend *shstatent; + + entry_ref = pgstat_prep_pending_entry(PGSTAT_KIND_BACKEND, InvalidOid, + procnum, NULL); + + shstatent = (PgStatShared_Backend *) entry_ref->shared_stats; + + /* + * NB: need to accept that there might be stats from an older backend, + * e.g. if we previously used this proc number. + */ + memset(&shstatent->stats, 0, sizeof(shstatent->stats)); +} + +/* + * Find or create a local PgStat_BackendPendingIO entry for procnum. + */ +PgStat_BackendPendingIO * +pgstat_prep_backend_pending(ProcNumber procnum) +{ + PgStat_EntryRef *entry_ref; + + entry_ref = pgstat_prep_pending_entry(PGSTAT_KIND_BACKEND, InvalidOid, + procnum, NULL); + + return entry_ref->pending; +} + +/* + * Backend statistics are not collected for all BackendTypes. + * + * The following BackendTypes do not participate in the backend stats + * subsystem: + * - The same and for the same reasons as in pgstat_tracks_io_bktype(). + * - B_BG_WRITER, B_CHECKPOINTER, B_STARTUP and B_AUTOVAC_LAUNCHER because their + * I/O stats are already visible in pg_stat_io and there is only one of those. + * + * Function returns true if BackendType participates in the backend stats + * subsystem for IO and false if it does not. + * + * When adding a new BackendType, also consider adding relevant restrictions to + * pgstat_tracks_io_object() and pgstat_tracks_io_op(). + */ +bool +pgstat_tracks_backend_bktype(BackendType bktype) +{ + /* + * List every type so that new backend types trigger a warning about + * needing to adjust this switch. + */ + switch (bktype) + { + case B_INVALID: + case B_AUTOVAC_LAUNCHER: + case B_DEAD_END_BACKEND: + case B_ARCHIVER: + case B_LOGGER: + case B_WAL_RECEIVER: + case B_WAL_WRITER: + case B_WAL_SUMMARIZER: + case B_BG_WRITER: + case B_CHECKPOINTER: + case B_STARTUP: + return false; + + case B_AUTOVAC_WORKER: + case B_BACKEND: + case B_BG_WORKER: + case B_STANDALONE_BACKEND: + case B_SLOTSYNC_WORKER: + case B_WAL_SENDER: + return true; + } + + return false; +} + +void +pgstat_backend_reset_timestamp_cb(PgStatShared_Common *header, TimestampTz ts) +{ + ((PgStatShared_Backend *) header)->stats.stat_reset_timestamp = ts; +} diff --git a/src/backend/utils/activity/pgstat_io.c b/src/backend/utils/activity/pgstat_io.c index f9883af2b3..e0c206a453 100644 --- a/src/backend/utils/activity/pgstat_io.c +++ b/src/backend/utils/activity/pgstat_io.c @@ -20,13 +20,7 @@ #include "storage/bufmgr.h" #include "utils/pgstat_internal.h" - -typedef struct PgStat_PendingIO -{ - PgStat_Counter counts[IOOBJECT_NUM_TYPES][IOCONTEXT_NUM_TYPES][IOOP_NUM_TYPES]; - instr_time pending_times[IOOBJECT_NUM_TYPES][IOCONTEXT_NUM_TYPES][IOOP_NUM_TYPES]; -} PgStat_PendingIO; - +typedef PgStat_BackendPendingIO PgStat_PendingIO; static PgStat_PendingIO PendingIOStats; static bool have_iostats = false; @@ -87,6 +81,14 @@ pgstat_count_io_op_n(IOObject io_object, IOContext io_context, IOOp io_op, uint3 Assert((unsigned int) io_op < IOOP_NUM_TYPES); Assert(pgstat_tracks_io_op(MyBackendType, io_object, io_context, io_op)); + if (pgstat_tracks_backend_bktype(MyBackendType)) + { + PgStat_PendingIO *entry_ref; + + entry_ref = pgstat_prep_backend_pending(MyProcNumber); + entry_ref->counts[io_object][io_context][io_op] += cnt; + } + PendingIOStats.counts[io_object][io_context][io_op] += cnt; have_iostats = true; @@ -148,6 +150,15 @@ pgstat_count_io_op_time(IOObject io_object, IOContext io_context, IOOp io_op, INSTR_TIME_ADD(PendingIOStats.pending_times[io_object][io_context][io_op], io_time); + + if (pgstat_tracks_backend_bktype(MyBackendType)) + { + PgStat_PendingIO *entry_ref; + + entry_ref = pgstat_prep_backend_pending(MyProcNumber); + INSTR_TIME_ADD(entry_ref->pending_times[io_object][io_context][io_op], + io_time); + } } pgstat_count_io_op_n(io_object, io_context, io_op, cnt); diff --git a/src/backend/utils/activity/pgstat_relation.c b/src/backend/utils/activity/pgstat_relation.c index faba8b64d2..85e65557bb 100644 --- a/src/backend/utils/activity/pgstat_relation.c +++ b/src/backend/utils/activity/pgstat_relation.c @@ -264,6 +264,7 @@ pgstat_report_vacuum(Oid tableoid, bool shared, * VACUUM command has processed all tables and committed. */ pgstat_flush_io(false); + pgstat_flush_backend(false); } /* @@ -350,6 +351,7 @@ pgstat_report_analyze(Relation rel, /* see pgstat_report_vacuum() */ pgstat_flush_io(false); + pgstat_flush_backend(false); } /* diff --git a/src/backend/utils/adt/pgstatfuncs.c b/src/backend/utils/adt/pgstatfuncs.c index cdf37403e9..b939551d36 100644 --- a/src/backend/utils/adt/pgstatfuncs.c +++ b/src/backend/utils/adt/pgstatfuncs.c @@ -1474,6 +1474,154 @@ pg_stat_get_io(PG_FUNCTION_ARGS) return (Datum) 0; } +Datum +pg_stat_get_backend_io(PG_FUNCTION_ARGS) +{ + ReturnSetInfo *rsinfo; + PgStat_Backend *backend_stats; + Datum bktype_desc; + PgStat_BktypeIO *bktype_stats; + BackendType bktype; + Datum reset_time; + int num_backends = pgstat_fetch_stat_numbackends(); + int curr_backend; + int pid; + PGPROC *proc; + ProcNumber procNumber; + + InitMaterializedSRF(fcinfo, 0); + rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; + + pid = PG_GETARG_INT32(0); + proc = BackendPidGetProc(pid); + + /* + * Could be an auxiliary process but would not report any stats due to + * pgstat_tracks_backend_bktype() anyway. So don't need an extra call to + * AuxiliaryPidGetProc(). + */ + if (!proc) + return (Datum) 0; + + procNumber = GetNumberFromPGProc(proc); + backend_stats = pgstat_fetch_proc_stat_io(procNumber); + + if (!backend_stats) + return (Datum) 0; + + bktype = B_INVALID; + + /* Look for the backend type */ + for (curr_backend = 1; curr_backend <= num_backends; curr_backend++) + { + LocalPgBackendStatus *local_beentry; + PgBackendStatus *beentry; + + /* Get the next one in the list */ + local_beentry = pgstat_get_local_beentry_by_index(curr_backend); + beentry = &local_beentry->backendStatus; + + /* Looking for specific PID, ignore all the others */ + if (beentry->st_procpid != pid) + continue; + + bktype = beentry->st_backendType; + break; + } + + /* Backend is gone */ + if (bktype == B_INVALID) + return (Datum) 0; + + bktype_desc = CStringGetTextDatum(GetBackendTypeDesc(bktype)); + bktype_stats = &backend_stats->stats; + reset_time = TimestampTzGetDatum(backend_stats->stat_reset_timestamp); + + /* + * In Assert builds, we can afford an extra loop through all of the + * counters checking that only expected stats are non-zero, since it keeps + * the non-Assert code cleaner. + */ + Assert(pgstat_bktype_io_stats_valid(bktype_stats, bktype)); + + for (int io_obj = 0; io_obj < IOOBJECT_NUM_TYPES; io_obj++) + { + const char *obj_name = pgstat_get_io_object_name(io_obj); + + for (int io_context = 0; io_context < IOCONTEXT_NUM_TYPES; io_context++) + { + const char *context_name = pgstat_get_io_context_name(io_context); + + Datum values[IO_NUM_COLUMNS] = {0}; + bool nulls[IO_NUM_COLUMNS] = {0}; + + /* + * Some combinations of BackendType, IOObject, and IOContext are + * not valid for any type of IOOp. In such cases, omit the entire + * row from the view. + */ + if (!pgstat_tracks_io_object(bktype, io_obj, io_context)) + continue; + + values[IO_COL_BACKEND_TYPE] = bktype_desc; + values[IO_COL_CONTEXT] = CStringGetTextDatum(context_name); + values[IO_COL_OBJECT] = CStringGetTextDatum(obj_name); + if (backend_stats->stat_reset_timestamp != 0) + values[IO_COL_RESET_TIME] = reset_time; + else + nulls[IO_COL_RESET_TIME] = true; + + /* + * Hard-code this to the value of BLCKSZ for now. Future values + * could include XLOG_BLCKSZ, once WAL IO is tracked, and constant + * multipliers, once non-block-oriented IO (e.g. temporary file + * IO) is tracked. + */ + values[IO_COL_CONVERSION] = Int64GetDatum(BLCKSZ); + + for (int io_op = 0; io_op < IOOP_NUM_TYPES; io_op++) + { + int op_idx = pgstat_get_io_op_index(io_op); + int time_idx = pgstat_get_io_time_index(io_op); + + /* + * Some combinations of BackendType and IOOp, of IOContext and + * IOOp, and of IOObject and IOOp are not tracked. Set these + * cells in the view NULL. + */ + if (pgstat_tracks_io_op(bktype, io_obj, io_context, io_op)) + { + PgStat_Counter count = + bktype_stats->counts[io_obj][io_context][io_op]; + + values[op_idx] = Int64GetDatum(count); + } + else + nulls[op_idx] = true; + + /* not every operation is timed */ + if (time_idx == IO_COL_INVALID) + continue; + + if (!nulls[op_idx]) + { + PgStat_Counter time = + bktype_stats->times[io_obj][io_context][io_op]; + + values[time_idx] = Float8GetDatum(pg_stat_us_to_ms(time)); + } + else + nulls[time_idx] = true; + } + + tuplestore_putvalues(rsinfo->setResult, rsinfo->setDesc, + values, nulls); + } + } + + return (Datum) 0; +} + /* * Returns statistics of WAL activity */ @@ -1779,6 +1927,27 @@ pg_stat_reset_single_function_counters(PG_FUNCTION_ARGS) PG_RETURN_VOID(); } +Datum +pg_stat_reset_backend_stats(PG_FUNCTION_ARGS) +{ + PGPROC *proc; + int backend_pid = PG_GETARG_INT32(0); + + proc = BackendPidGetProc(backend_pid); + + /* + * Could be an auxiliary process but would not report any stats due to + * pgstat_tracks_backend_bktype() anyway. So don't need an extra call to + * AuxiliaryPidGetProc(). + */ + if (!proc) + PG_RETURN_VOID(); + + pgstat_reset(PGSTAT_KIND_BACKEND, InvalidOid, GetNumberFromPGProc(proc)); + + PG_RETURN_VOID(); +} + /* Reset SLRU counters (a specific one or all of them). */ Datum pg_stat_reset_slru(PG_FUNCTION_ARGS) diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index 0f22c21723..437157ffa3 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -5913,6 +5913,15 @@ proargnames => '{backend_type,object,context,reads,read_time,writes,write_time,writebacks,writeback_time,extends,extend_time,op_bytes,hits,evictions,reuses,fsyncs,fsync_time,stats_reset}', prosrc => 'pg_stat_get_io' }, +{ oid => '8806', descr => 'statistics: backend IO statistics', + proname => 'pg_stat_get_backend_io', prorows => '5', proretset => 't', + provolatile => 'v', proparallel => 'r', prorettype => 'record', + proargtypes => 'int4', + proallargtypes => '{int4,text,text,text,int8,float8,int8,float8,int8,float8,int8,float8,int8,int8,int8,int8,int8,float8,timestamptz}', + proargmodes => '{i,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o}', + proargnames => '{backend_pid,backend_type,object,context,reads,read_time,writes,write_time,writebacks,writeback_time,extends,extend_time,op_bytes,hits,evictions,reuses,fsyncs,fsync_time,stats_reset}', + prosrc => 'pg_stat_get_backend_io' }, + { oid => '1136', descr => 'statistics: information about WAL activity', proname => 'pg_stat_get_wal', proisstrict => 'f', provolatile => 's', proparallel => 'r', prorettype => 'record', proargtypes => '', @@ -6052,6 +6061,11 @@ proname => 'pg_stat_reset_single_function_counters', provolatile => 'v', prorettype => 'void', proargtypes => 'oid', prosrc => 'pg_stat_reset_single_function_counters' }, +{ oid => '9987', + descr => 'statistics: reset statistics for a single backend', + proname => 'pg_stat_reset_backend_stats', provolatile => 'v', + prorettype => 'void', proargtypes => 'int4', + prosrc => 'pg_stat_reset_backend_stats' }, { oid => '2307', descr => 'statistics: reset collected statistics for a single SLRU', proname => 'pg_stat_reset_slru', proisstrict => 'f', provolatile => 'v', diff --git a/src/include/pgstat.h b/src/include/pgstat.h index ebfeef2f46..479773cfd2 100644 --- a/src/include/pgstat.h +++ b/src/include/pgstat.h @@ -49,14 +49,15 @@ #define PGSTAT_KIND_FUNCTION 3 /* per-function statistics */ #define PGSTAT_KIND_REPLSLOT 4 /* per-slot statistics */ #define PGSTAT_KIND_SUBSCRIPTION 5 /* per-subscription statistics */ +#define PGSTAT_KIND_BACKEND 6 /* per-backend statistics */ /* stats for fixed-numbered objects */ -#define PGSTAT_KIND_ARCHIVER 6 -#define PGSTAT_KIND_BGWRITER 7 -#define PGSTAT_KIND_CHECKPOINTER 8 -#define PGSTAT_KIND_IO 9 -#define PGSTAT_KIND_SLRU 10 -#define PGSTAT_KIND_WAL 11 +#define PGSTAT_KIND_ARCHIVER 7 +#define PGSTAT_KIND_BGWRITER 8 +#define PGSTAT_KIND_CHECKPOINTER 9 +#define PGSTAT_KIND_IO 10 +#define PGSTAT_KIND_SLRU 11 +#define PGSTAT_KIND_WAL 12 #define PGSTAT_KIND_BUILTIN_MIN PGSTAT_KIND_DATABASE #define PGSTAT_KIND_BUILTIN_MAX PGSTAT_KIND_WAL @@ -362,12 +363,23 @@ typedef struct PgStat_BktypeIO PgStat_Counter times[IOOBJECT_NUM_TYPES][IOCONTEXT_NUM_TYPES][IOOP_NUM_TYPES]; } PgStat_BktypeIO; +typedef struct PgStat_BackendPendingIO +{ + PgStat_Counter counts[IOOBJECT_NUM_TYPES][IOCONTEXT_NUM_TYPES][IOOP_NUM_TYPES]; + instr_time pending_times[IOOBJECT_NUM_TYPES][IOCONTEXT_NUM_TYPES][IOOP_NUM_TYPES]; +} PgStat_BackendPendingIO; + typedef struct PgStat_IO { TimestampTz stat_reset_timestamp; PgStat_BktypeIO stats[BACKEND_NUM_TYPES]; } PgStat_IO; +typedef struct PgStat_Backend +{ + TimestampTz stat_reset_timestamp; + PgStat_BktypeIO stats; +} PgStat_Backend; typedef struct PgStat_StatDBEntry { @@ -549,6 +561,13 @@ extern bool pgstat_have_entry(PgStat_Kind kind, Oid dboid, uint64 objid); extern void pgstat_report_archiver(const char *xlog, bool failed); extern PgStat_ArchiverStats *pgstat_fetch_stat_archiver(void); +/* + * Functions in pgstat_backend.c + */ + +extern PgStat_Backend *pgstat_fetch_proc_stat_io(ProcNumber procNumber); +extern bool pgstat_tracks_backend_bktype(BackendType bktype); +extern void pgstat_create_backend_stat(ProcNumber procnum); /* * Functions in pgstat_bgwriter.c diff --git a/src/include/utils/pgstat_internal.h b/src/include/utils/pgstat_internal.h index 7338bc1e28..811ed9b005 100644 --- a/src/include/utils/pgstat_internal.h +++ b/src/include/utils/pgstat_internal.h @@ -450,6 +450,11 @@ typedef struct PgStatShared_ReplSlot PgStat_StatReplSlotEntry stats; } PgStatShared_ReplSlot; +typedef struct PgStatShared_Backend +{ + PgStatShared_Common header; + PgStat_Backend stats; +} PgStatShared_Backend; /* * Central shared memory entry for the cumulative stats system. @@ -604,6 +609,15 @@ extern void pgstat_archiver_init_shmem_cb(void *stats); extern void pgstat_archiver_reset_all_cb(TimestampTz ts); extern void pgstat_archiver_snapshot_cb(void); +/* + * Functions in pgstat_backend.c + */ + +extern void pgstat_flush_backend(bool nowait); + +extern PgStat_BackendPendingIO *pgstat_prep_backend_pending(ProcNumber procnum); +extern bool pgstat_backend_flush_cb(PgStat_EntryRef *entry_ref, bool nowait); +extern void pgstat_backend_reset_timestamp_cb(PgStatShared_Common *header, TimestampTz ts); /* * Functions in pgstat_bgwriter.c diff --git a/src/test/regress/expected/stats.out b/src/test/regress/expected/stats.out index 56771f83ed..3447e7b75d 100644 --- a/src/test/regress/expected/stats.out +++ b/src/test/regress/expected/stats.out @@ -1249,7 +1249,7 @@ SELECT pg_stat_get_subscription_stats(NULL); (1 row) --- Test that the following operations are tracked in pg_stat_io: +-- Test that the following operations are tracked in pg_stat_io and in backend stats: -- - reads of target blocks into shared buffers -- - writes of shared buffers to permanent storage -- - extends of relations using shared buffers @@ -1259,11 +1259,19 @@ SELECT pg_stat_get_subscription_stats(NULL); -- be sure of the state of shared buffers at the point the test is run. -- Create a regular table and insert some data to generate IOCONTEXT_NORMAL -- extends. +SELECT pid AS checkpointer_pid FROM pg_stat_activity + WHERE backend_type = 'checkpointer' \gset SELECT sum(extends) AS io_sum_shared_before_extends FROM pg_stat_io WHERE context = 'normal' AND object = 'relation' \gset +SELECT sum(extends) AS my_io_sum_shared_before_extends + FROM pg_stat_get_backend_io(pg_backend_pid()) + WHERE context = 'normal' AND object = 'relation' \gset SELECT sum(writes) AS writes, sum(fsyncs) AS fsyncs FROM pg_stat_io WHERE object = 'relation' \gset io_sum_shared_before_ +SELECT sum(writes) AS writes, sum(fsyncs) AS fsyncs + FROM pg_stat_get_backend_io(pg_backend_pid()) + WHERE object = 'relation' \gset my_io_sum_shared_before_ CREATE TABLE test_io_shared(a int); INSERT INTO test_io_shared SELECT i FROM generate_series(1,100)i; SELECT pg_stat_force_next_flush(); @@ -1280,8 +1288,17 @@ SELECT :io_sum_shared_after_extends > :io_sum_shared_before_extends; t (1 row) +SELECT sum(extends) AS my_io_sum_shared_after_extends + FROM pg_stat_get_backend_io(pg_backend_pid()) + WHERE context = 'normal' AND object = 'relation' \gset +SELECT :my_io_sum_shared_after_extends > :my_io_sum_shared_before_extends; + ?column? +---------- + t +(1 row) + -- After a checkpoint, there should be some additional IOCONTEXT_NORMAL writes --- and fsyncs. +-- and fsyncs in the global stats (not for the backend). -- See comment above for rationale for two explicit CHECKPOINTs. CHECKPOINT; CHECKPOINT; @@ -1301,6 +1318,31 @@ SELECT current_setting('fsync') = 'off' t (1 row) +SELECT sum(writes) AS writes, sum(fsyncs) AS fsyncs + FROM pg_stat_get_backend_io(pg_backend_pid()) + WHERE object = 'relation' \gset my_io_sum_shared_after_ +SELECT :my_io_sum_shared_after_writes >= :my_io_sum_shared_before_writes; + ?column? +---------- + t +(1 row) + +SELECT current_setting('fsync') = 'off' + OR (:my_io_sum_shared_after_fsyncs = :my_io_sum_shared_before_fsyncs + AND :my_io_sum_shared_after_fsyncs= 0); + ?column? +---------- + t +(1 row) + +-- Don't return any rows if querying other backend's stats that are excluded +-- from the backend stats collection (like the checkpointer). +SELECT count(1) = 0 FROM pg_stat_get_backend_io(:checkpointer_pid); + ?column? +---------- + t +(1 row) + -- Change the tablespace so that the table is rewritten directly, then SELECT -- from it to cause it to be read back into shared buffers. SELECT sum(reads) AS io_sum_shared_before_reads @@ -1521,6 +1563,8 @@ SELECT pg_stat_have_stats('io', 0, 0); SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS io_stats_pre_reset FROM pg_stat_io \gset +SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS my_io_stats_pre_reset + FROM pg_stat_get_backend_io(pg_backend_pid()) \gset SELECT pg_stat_reset_shared('io'); pg_stat_reset_shared ---------------------- @@ -1535,6 +1579,30 @@ SELECT :io_stats_post_reset < :io_stats_pre_reset; t (1 row) +SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS my_io_stats_post_reset + FROM pg_stat_get_backend_io(pg_backend_pid()) \gset +-- pg_stat_reset_shared() did not reset backend IO stats +SELECT :my_io_stats_pre_reset <= :my_io_stats_post_reset; + ?column? +---------- + t +(1 row) + +-- but pg_stat_reset_backend_stats() does +SELECT pg_stat_reset_backend_stats(pg_backend_pid()); + pg_stat_reset_backend_stats +----------------------------- + +(1 row) + +SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS my_io_stats_post_backend_reset + FROM pg_stat_get_backend_io(pg_backend_pid()) \gset +SELECT :my_io_stats_pre_reset > :my_io_stats_post_backend_reset; + ?column? +---------- + t +(1 row) + -- test BRIN index doesn't block HOT update CREATE TABLE brin_hot ( id integer PRIMARY KEY, diff --git a/src/test/regress/sql/stats.sql b/src/test/regress/sql/stats.sql index 7147cc2f89..9c925005be 100644 --- a/src/test/regress/sql/stats.sql +++ b/src/test/regress/sql/stats.sql @@ -595,7 +595,7 @@ SELECT pg_stat_get_replication_slot(NULL); SELECT pg_stat_get_subscription_stats(NULL); --- Test that the following operations are tracked in pg_stat_io: +-- Test that the following operations are tracked in pg_stat_io and in backend stats: -- - reads of target blocks into shared buffers -- - writes of shared buffers to permanent storage -- - extends of relations using shared buffers @@ -607,20 +607,32 @@ SELECT pg_stat_get_subscription_stats(NULL); -- Create a regular table and insert some data to generate IOCONTEXT_NORMAL -- extends. +SELECT pid AS checkpointer_pid FROM pg_stat_activity + WHERE backend_type = 'checkpointer' \gset SELECT sum(extends) AS io_sum_shared_before_extends FROM pg_stat_io WHERE context = 'normal' AND object = 'relation' \gset +SELECT sum(extends) AS my_io_sum_shared_before_extends + FROM pg_stat_get_backend_io(pg_backend_pid()) + WHERE context = 'normal' AND object = 'relation' \gset SELECT sum(writes) AS writes, sum(fsyncs) AS fsyncs FROM pg_stat_io WHERE object = 'relation' \gset io_sum_shared_before_ +SELECT sum(writes) AS writes, sum(fsyncs) AS fsyncs + FROM pg_stat_get_backend_io(pg_backend_pid()) + WHERE object = 'relation' \gset my_io_sum_shared_before_ CREATE TABLE test_io_shared(a int); INSERT INTO test_io_shared SELECT i FROM generate_series(1,100)i; SELECT pg_stat_force_next_flush(); SELECT sum(extends) AS io_sum_shared_after_extends FROM pg_stat_io WHERE context = 'normal' AND object = 'relation' \gset SELECT :io_sum_shared_after_extends > :io_sum_shared_before_extends; +SELECT sum(extends) AS my_io_sum_shared_after_extends + FROM pg_stat_get_backend_io(pg_backend_pid()) + WHERE context = 'normal' AND object = 'relation' \gset +SELECT :my_io_sum_shared_after_extends > :my_io_sum_shared_before_extends; -- After a checkpoint, there should be some additional IOCONTEXT_NORMAL writes --- and fsyncs. +-- and fsyncs in the global stats (not for the backend). -- See comment above for rationale for two explicit CHECKPOINTs. CHECKPOINT; CHECKPOINT; @@ -630,6 +642,17 @@ SELECT sum(writes) AS writes, sum(fsyncs) AS fsyncs SELECT :io_sum_shared_after_writes > :io_sum_shared_before_writes; SELECT current_setting('fsync') = 'off' OR :io_sum_shared_after_fsyncs > :io_sum_shared_before_fsyncs; +SELECT sum(writes) AS writes, sum(fsyncs) AS fsyncs + FROM pg_stat_get_backend_io(pg_backend_pid()) + WHERE object = 'relation' \gset my_io_sum_shared_after_ +SELECT :my_io_sum_shared_after_writes >= :my_io_sum_shared_before_writes; +SELECT current_setting('fsync') = 'off' + OR (:my_io_sum_shared_after_fsyncs = :my_io_sum_shared_before_fsyncs + AND :my_io_sum_shared_after_fsyncs= 0); + +-- Don't return any rows if querying other backend's stats that are excluded +-- from the backend stats collection (like the checkpointer). +SELECT count(1) = 0 FROM pg_stat_get_backend_io(:checkpointer_pid); -- Change the tablespace so that the table is rewritten directly, then SELECT -- from it to cause it to be read back into shared buffers. @@ -762,10 +785,21 @@ SELECT :io_sum_bulkwrite_strategy_extends_after > :io_sum_bulkwrite_strategy_ext SELECT pg_stat_have_stats('io', 0, 0); SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS io_stats_pre_reset FROM pg_stat_io \gset +SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS my_io_stats_pre_reset + FROM pg_stat_get_backend_io(pg_backend_pid()) \gset SELECT pg_stat_reset_shared('io'); SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS io_stats_post_reset FROM pg_stat_io \gset SELECT :io_stats_post_reset < :io_stats_pre_reset; +SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS my_io_stats_post_reset + FROM pg_stat_get_backend_io(pg_backend_pid()) \gset +-- pg_stat_reset_shared() did not reset backend IO stats +SELECT :my_io_stats_pre_reset <= :my_io_stats_post_reset; +-- but pg_stat_reset_backend_stats() does +SELECT pg_stat_reset_backend_stats(pg_backend_pid()); +SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS my_io_stats_post_backend_reset + FROM pg_stat_get_backend_io(pg_backend_pid()) \gset +SELECT :my_io_stats_pre_reset > :my_io_stats_post_backend_reset; -- test BRIN index doesn't block HOT update diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index ce33e55bf1..398dd92527 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2121,6 +2121,7 @@ PgFdwSamplingMethod PgFdwScanState PgIfAddrCallback PgStatShared_Archiver +PgStatShared_Backend PgStatShared_BgWriter PgStatShared_Checkpointer PgStatShared_Common @@ -2136,6 +2137,8 @@ PgStatShared_SLRU PgStatShared_Subscription PgStatShared_Wal PgStat_ArchiverStats +PgStat_Backend +PgStat_BackendPendingIO PgStat_BackendSubEntry PgStat_BgWriterStats PgStat_BktypeIO -- 2.34.1 --Sc8S/0qAVNqKo8ol-- ^ permalink raw reply [nested|flat] 25+ messages in thread
* [PATCH v9] per backend I/O statistics @ 2024-10-28 12:50 Bertrand Drouvot <[email protected]> 0 siblings, 0 replies; 25+ messages in thread From: Bertrand Drouvot @ 2024-10-28 12:50 UTC (permalink / raw) While pg_stat_io provides cluster-wide I/O statistics, this commit adds the ability to track and display per backend I/O statistics. It adds a new statistics kind and 2 new functions: - pg_stat_reset_backend_stats() to be able to reset the stats for a given backend pid. - pg_stat_get_backend_io() to retrieve I/O statistics for a given backend pid. The new KIND is named PGSTAT_KIND_BACKEND as it could be used in the future to store other statistics (than the I/O ones) per backend. The new KIND is a variable-numbered one and has an automatic cap on the maximum number of entries (as its hash key contains the proc number). There is no need to write the per backend I/O stats to disk (no point to see stats for backends that do not exist anymore after a re-start), so using "write_to_file = false". Note that per backend I/O statistics are not collected for the checkpointer, the background writer, the startup process and the autovacuum launcher as those are already visible in pg_stat_io and there is only one of those. XXX: Bump catalog version needs to be done. --- doc/src/sgml/config.sgml | 8 +- doc/src/sgml/monitoring.sgml | 37 ++++ src/backend/catalog/system_functions.sql | 2 + src/backend/utils/activity/Makefile | 1 + src/backend/utils/activity/backend_status.c | 4 + src/backend/utils/activity/meson.build | 1 + src/backend/utils/activity/pgstat.c | 23 ++- src/backend/utils/activity/pgstat_backend.c | 182 +++++++++++++++++++ src/backend/utils/activity/pgstat_io.c | 25 ++- src/backend/utils/activity/pgstat_relation.c | 2 + src/backend/utils/adt/pgstatfuncs.c | 169 +++++++++++++++++ src/include/catalog/pg_proc.dat | 14 ++ src/include/pgstat.h | 31 +++- src/include/utils/pgstat_internal.h | 14 ++ src/test/regress/expected/stats.out | 72 +++++++- src/test/regress/sql/stats.sql | 38 +++- src/tools/pgindent/typedefs.list | 3 + 17 files changed, 605 insertions(+), 21 deletions(-) 9.9% doc/src/sgml/ 31.3% src/backend/utils/activity/ 22.0% src/backend/utils/adt/ 4.4% src/include/catalog/ 7.0% src/include/ 12.8% src/test/regress/expected/ 11.5% src/test/regress/sql/ diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index e0c8325a39..8afca9b110 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -8403,9 +8403,11 @@ COPY postgres_log FROM '/full/path/to/logfile.csv' WITH csv; displayed in <link linkend="monitoring-pg-stat-database-view"> <structname>pg_stat_database</structname></link>, <link linkend="monitoring-pg-stat-io-view"> - <structname>pg_stat_io</structname></link>, in the output of - <xref linkend="sql-explain"/> when the <literal>BUFFERS</literal> option - is used, in the output of <xref linkend="sql-vacuum"/> when + <structname>pg_stat_io</structname></link>, in the output of the + <link linkend="pg-stat-get-backend-io"> + <function>pg_stat_get_backend_io()</function></link> function, in the + output of <xref linkend="sql-explain"/> when the <literal>BUFFERS</literal> + option is used, in the output of <xref linkend="sql-vacuum"/> when the <literal>VERBOSE</literal> option is used, by autovacuum for auto-vacuums and auto-analyzes, when <xref linkend="guc-log-autovacuum-min-duration"/> is set and by diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml index 840d7f8161..4a9464da61 100644 --- a/doc/src/sgml/monitoring.sgml +++ b/doc/src/sgml/monitoring.sgml @@ -4790,6 +4790,25 @@ description | Waiting for a newly initialized WAL file to reach durable storage </para></entry> </row> + <row> + <entry id="pg-stat-get-backend-io" role="func_table_entry"><para role="func_signature"> + <indexterm> + <primary>pg_stat_get_backend_io</primary> + </indexterm> + <function>pg_stat_get_backend_io</function> ( <type>integer</type> ) + <returnvalue>setof record</returnvalue> + </para> + <para> + Returns I/O statistics about the backend with the specified + process ID. The output fields are exactly the same as the ones in the + <link linkend="monitoring-pg-stat-io-view"> <structname>pg_stat_io</structname></link> + view. The function does not return I/O statistics for the checkpointer, + the background writer, the startup process and the autovacuum launcher + as they are already visible in the <link linkend="monitoring-pg-stat-io-view"> <structname>pg_stat_io</structname></link> + view and there is only one of those. + </para></entry> + </row> + <row> <entry role="func_table_entry"><para role="func_signature"> <indexterm> @@ -4971,6 +4990,24 @@ description | Waiting for a newly initialized WAL file to reach durable storage </para></entry> </row> + <row> + <entry role="func_table_entry"><para role="func_signature"> + <indexterm> + <primary>pg_stat_reset_backend_stats</primary> + </indexterm> + <function>pg_stat_reset_backend_stats</function> ( <type>integer</type> ) + <returnvalue>void</returnvalue> + </para> + <para> + Resets statistics for a single backend with the specified process ID + to zero. + </para> + <para> + This function is restricted to superusers by default, but other users + can be granted EXECUTE to run the function. + </para></entry> + </row> + <row> <entry role="func_table_entry"><para role="func_signature"> <indexterm> diff --git a/src/backend/catalog/system_functions.sql b/src/backend/catalog/system_functions.sql index c51dfca802..14eb99cd47 100644 --- a/src/backend/catalog/system_functions.sql +++ b/src/backend/catalog/system_functions.sql @@ -711,6 +711,8 @@ REVOKE EXECUTE ON FUNCTION pg_stat_reset_single_table_counters(oid) FROM public; REVOKE EXECUTE ON FUNCTION pg_stat_reset_single_function_counters(oid) FROM public; +REVOKE EXECUTE ON FUNCTION pg_stat_reset_backend_stats(integer) FROM public; + REVOKE EXECUTE ON FUNCTION pg_stat_reset_replication_slot(text) FROM public; REVOKE EXECUTE ON FUNCTION pg_stat_have_stats(text, oid, int8) FROM public; diff --git a/src/backend/utils/activity/Makefile b/src/backend/utils/activity/Makefile index b9fd66ea17..24b64a2742 100644 --- a/src/backend/utils/activity/Makefile +++ b/src/backend/utils/activity/Makefile @@ -20,6 +20,7 @@ OBJS = \ backend_status.o \ pgstat.o \ pgstat_archiver.o \ + pgstat_backend.o \ pgstat_bgwriter.o \ pgstat_checkpointer.o \ pgstat_database.o \ diff --git a/src/backend/utils/activity/backend_status.c b/src/backend/utils/activity/backend_status.c index 22c6dc378c..e9c3b87ac2 100644 --- a/src/backend/utils/activity/backend_status.c +++ b/src/backend/utils/activity/backend_status.c @@ -426,6 +426,10 @@ pgstat_bestart(void) PGSTAT_END_WRITE_ACTIVITY(vbeentry); + /* Create the backend statistics entry */ + if (pgstat_tracks_backend_bktype(MyBackendType)) + pgstat_create_backend_stat(MyProcNumber); + /* Update app name to current GUC setting */ if (application_name) pgstat_report_appname(application_name); diff --git a/src/backend/utils/activity/meson.build b/src/backend/utils/activity/meson.build index f73c22905c..380d3dd70c 100644 --- a/src/backend/utils/activity/meson.build +++ b/src/backend/utils/activity/meson.build @@ -5,6 +5,7 @@ backend_sources += files( 'backend_status.c', 'pgstat.c', 'pgstat_archiver.c', + 'pgstat_backend.c', 'pgstat_bgwriter.c', 'pgstat_checkpointer.c', 'pgstat_database.c', diff --git a/src/backend/utils/activity/pgstat.c b/src/backend/utils/activity/pgstat.c index 18b7d9b47d..9dd9d60b00 100644 --- a/src/backend/utils/activity/pgstat.c +++ b/src/backend/utils/activity/pgstat.c @@ -77,6 +77,7 @@ * * Each statistics kind is handled in a dedicated file: * - pgstat_archiver.c + * - pgstat_backend.c * - pgstat_bgwriter.c * - pgstat_checkpointer.c * - pgstat_database.c @@ -358,6 +359,22 @@ static const PgStat_KindInfo pgstat_kind_builtin_infos[PGSTAT_KIND_BUILTIN_SIZE] .reset_timestamp_cb = pgstat_subscription_reset_timestamp_cb, }, + [PGSTAT_KIND_BACKEND] = { + .name = "backend", + + .fixed_amount = false, + .write_to_file = false, + + .accessed_across_databases = true, + + .shared_size = sizeof(PgStatShared_Backend), + .shared_data_off = offsetof(PgStatShared_Backend, stats), + .shared_data_len = sizeof(((PgStatShared_Backend *) 0)->stats), + .pending_size = sizeof(PgStat_BackendPendingIO), + + .flush_pending_cb = pgstat_backend_flush_cb, + .reset_timestamp_cb = pgstat_backend_reset_timestamp_cb, + }, /* stats for fixed-numbered (mostly 1) objects */ @@ -602,6 +619,10 @@ pgstat_shutdown_hook(int code, Datum arg) Assert(dlist_is_empty(&pgStatPending)); dlist_init(&pgStatPending); + /* drop the backend stats entry */ + if (!pgstat_drop_entry(PGSTAT_KIND_BACKEND, InvalidOid, MyProcNumber)) + pgstat_request_entry_refs_gc(); + pgstat_detach_shmem(); #ifdef USE_ASSERT_CHECKING @@ -768,7 +789,7 @@ pgstat_report_stat(bool force) partial_flush = false; - /* flush database / relation / function / ... stats */ + /* flush database / relation / function / backend / ... stats */ partial_flush |= pgstat_flush_pending_entries(nowait); /* flush of fixed-numbered stats */ diff --git a/src/backend/utils/activity/pgstat_backend.c b/src/backend/utils/activity/pgstat_backend.c new file mode 100644 index 0000000000..e2d83024c2 --- /dev/null +++ b/src/backend/utils/activity/pgstat_backend.c @@ -0,0 +1,182 @@ +/* ------------------------------------------------------------------------- + * + * pgstat_backend.c + * Implementation of backend statistics. + * + * This file contains the implementation of backend statistics. It is kept + * separate from pgstat.c to enforce the line between the statistics access / + * storage implementation and the details about individual types of statistics. + * + * Copyright (c) 2024, PostgreSQL Global Development Group + * + * IDENTIFICATION + * src/backend/utils/activity/pgstat_backend.c + * ------------------------------------------------------------------------- + */ + +#include "postgres.h" + +#include "utils/pgstat_internal.h" + +/* + * Returns backend's IO stats. + */ +PgStat_Backend * +pgstat_fetch_proc_stat_io(ProcNumber procNumber) +{ + PgStat_Backend *backend_entry; + + backend_entry = (PgStat_Backend *) pgstat_fetch_entry(PGSTAT_KIND_BACKEND, + InvalidOid, procNumber); + + return backend_entry; +} + +/* + * Flush out locally pending backend statistics + * + * If no stats have been recorded, this function returns false. + */ +bool +pgstat_backend_flush_cb(PgStat_EntryRef *entry_ref, bool nowait) +{ + PgStatShared_Backend *shbackendioent; + PgStat_BackendPendingIO *pendingent; + PgStat_BktypeIO *bktype_shstats; + + if (!pgstat_lock_entry(entry_ref, nowait)) + return false; + + shbackendioent = (PgStatShared_Backend *) entry_ref->shared_stats; + bktype_shstats = &shbackendioent->stats.stats; + pendingent = (PgStat_BackendPendingIO *) entry_ref->pending; + + for (int io_object = 0; io_object < IOOBJECT_NUM_TYPES; io_object++) + { + for (int io_context = 0; io_context < IOCONTEXT_NUM_TYPES; io_context++) + { + for (int io_op = 0; io_op < IOOP_NUM_TYPES; io_op++) + { + instr_time time; + + bktype_shstats->counts[io_object][io_context][io_op] += + pendingent->counts[io_object][io_context][io_op]; + + time = pendingent->pending_times[io_object][io_context][io_op]; + + bktype_shstats->times[io_object][io_context][io_op] += + INSTR_TIME_GET_MICROSEC(time); + } + } + } + + pgstat_unlock_entry(entry_ref); + + return true; +} + +/* + * Simpler wrapper of pgstat_backend_flush_cb() + */ +void +pgstat_flush_backend(bool nowait) +{ + if (pgstat_tracks_backend_bktype(MyBackendType)) + { + PgStat_EntryRef *entry_ref; + + entry_ref = pgstat_get_entry_ref(PGSTAT_KIND_BACKEND, InvalidOid, + MyProcNumber, false, NULL); + (void) pgstat_backend_flush_cb(entry_ref, nowait); + } +} + +/* + * Create the backend statistics entry for procnum. + */ +void +pgstat_create_backend_stat(ProcNumber procnum) +{ + PgStat_EntryRef *entry_ref; + PgStatShared_Backend *shstatent; + + entry_ref = pgstat_prep_pending_entry(PGSTAT_KIND_BACKEND, InvalidOid, + procnum, NULL); + + shstatent = (PgStatShared_Backend *) entry_ref->shared_stats; + + /* + * NB: need to accept that there might be stats from an older backend, + * e.g. if we previously used this proc number. + */ + memset(&shstatent->stats, 0, sizeof(shstatent->stats)); +} + +/* + * Find or create a local PgStat_BackendPendingIO entry for procnum. + */ +PgStat_BackendPendingIO * +pgstat_prep_backend_pending(ProcNumber procnum) +{ + PgStat_EntryRef *entry_ref; + + entry_ref = pgstat_prep_pending_entry(PGSTAT_KIND_BACKEND, InvalidOid, + procnum, NULL); + + return entry_ref->pending; +} + +/* + * Backend statistics are not collected for all BackendTypes. + * + * The following BackendTypes do not participate in the backend stats + * subsystem: + * - The same and for the same reasons as in pgstat_tracks_io_bktype(). + * - B_BG_WRITER, B_CHECKPOINTER, B_STARTUP and B_AUTOVAC_LAUNCHER because their + * I/O stats are already visible in pg_stat_io and there is only one of those. + * + * Function returns true if BackendType participates in the backend stats + * subsystem for IO and false if it does not. + * + * When adding a new BackendType, also consider adding relevant restrictions to + * pgstat_tracks_io_object() and pgstat_tracks_io_op(). + */ +bool +pgstat_tracks_backend_bktype(BackendType bktype) +{ + /* + * List every type so that new backend types trigger a warning about + * needing to adjust this switch. + */ + switch (bktype) + { + case B_INVALID: + case B_AUTOVAC_LAUNCHER: + case B_DEAD_END_BACKEND: + case B_ARCHIVER: + case B_LOGGER: + case B_WAL_RECEIVER: + case B_WAL_WRITER: + case B_WAL_SUMMARIZER: + case B_BG_WRITER: + case B_CHECKPOINTER: + case B_STARTUP: + return false; + + case B_AUTOVAC_WORKER: + case B_BACKEND: + case B_BG_WORKER: + case B_STANDALONE_BACKEND: + case B_SLOTSYNC_WORKER: + case B_WAL_SENDER: + return true; + } + + return false; +} + +void +pgstat_backend_reset_timestamp_cb(PgStatShared_Common *header, TimestampTz ts) +{ + ((PgStatShared_Backend *) header)->stats.stat_reset_timestamp = ts; +} diff --git a/src/backend/utils/activity/pgstat_io.c b/src/backend/utils/activity/pgstat_io.c index f9883af2b3..e0c206a453 100644 --- a/src/backend/utils/activity/pgstat_io.c +++ b/src/backend/utils/activity/pgstat_io.c @@ -20,13 +20,7 @@ #include "storage/bufmgr.h" #include "utils/pgstat_internal.h" - -typedef struct PgStat_PendingIO -{ - PgStat_Counter counts[IOOBJECT_NUM_TYPES][IOCONTEXT_NUM_TYPES][IOOP_NUM_TYPES]; - instr_time pending_times[IOOBJECT_NUM_TYPES][IOCONTEXT_NUM_TYPES][IOOP_NUM_TYPES]; -} PgStat_PendingIO; - +typedef PgStat_BackendPendingIO PgStat_PendingIO; static PgStat_PendingIO PendingIOStats; static bool have_iostats = false; @@ -87,6 +81,14 @@ pgstat_count_io_op_n(IOObject io_object, IOContext io_context, IOOp io_op, uint3 Assert((unsigned int) io_op < IOOP_NUM_TYPES); Assert(pgstat_tracks_io_op(MyBackendType, io_object, io_context, io_op)); + if (pgstat_tracks_backend_bktype(MyBackendType)) + { + PgStat_PendingIO *entry_ref; + + entry_ref = pgstat_prep_backend_pending(MyProcNumber); + entry_ref->counts[io_object][io_context][io_op] += cnt; + } + PendingIOStats.counts[io_object][io_context][io_op] += cnt; have_iostats = true; @@ -148,6 +150,15 @@ pgstat_count_io_op_time(IOObject io_object, IOContext io_context, IOOp io_op, INSTR_TIME_ADD(PendingIOStats.pending_times[io_object][io_context][io_op], io_time); + + if (pgstat_tracks_backend_bktype(MyBackendType)) + { + PgStat_PendingIO *entry_ref; + + entry_ref = pgstat_prep_backend_pending(MyProcNumber); + INSTR_TIME_ADD(entry_ref->pending_times[io_object][io_context][io_op], + io_time); + } } pgstat_count_io_op_n(io_object, io_context, io_op, cnt); diff --git a/src/backend/utils/activity/pgstat_relation.c b/src/backend/utils/activity/pgstat_relation.c index faba8b64d2..85e65557bb 100644 --- a/src/backend/utils/activity/pgstat_relation.c +++ b/src/backend/utils/activity/pgstat_relation.c @@ -264,6 +264,7 @@ pgstat_report_vacuum(Oid tableoid, bool shared, * VACUUM command has processed all tables and committed. */ pgstat_flush_io(false); + pgstat_flush_backend(false); } /* @@ -350,6 +351,7 @@ pgstat_report_analyze(Relation rel, /* see pgstat_report_vacuum() */ pgstat_flush_io(false); + pgstat_flush_backend(false); } /* diff --git a/src/backend/utils/adt/pgstatfuncs.c b/src/backend/utils/adt/pgstatfuncs.c index cdf37403e9..b939551d36 100644 --- a/src/backend/utils/adt/pgstatfuncs.c +++ b/src/backend/utils/adt/pgstatfuncs.c @@ -1474,6 +1474,154 @@ pg_stat_get_io(PG_FUNCTION_ARGS) return (Datum) 0; } +Datum +pg_stat_get_backend_io(PG_FUNCTION_ARGS) +{ + ReturnSetInfo *rsinfo; + PgStat_Backend *backend_stats; + Datum bktype_desc; + PgStat_BktypeIO *bktype_stats; + BackendType bktype; + Datum reset_time; + int num_backends = pgstat_fetch_stat_numbackends(); + int curr_backend; + int pid; + PGPROC *proc; + ProcNumber procNumber; + + InitMaterializedSRF(fcinfo, 0); + rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; + + pid = PG_GETARG_INT32(0); + proc = BackendPidGetProc(pid); + + /* + * Could be an auxiliary process but would not report any stats due to + * pgstat_tracks_backend_bktype() anyway. So don't need an extra call to + * AuxiliaryPidGetProc(). + */ + if (!proc) + return (Datum) 0; + + procNumber = GetNumberFromPGProc(proc); + backend_stats = pgstat_fetch_proc_stat_io(procNumber); + + if (!backend_stats) + return (Datum) 0; + + bktype = B_INVALID; + + /* Look for the backend type */ + for (curr_backend = 1; curr_backend <= num_backends; curr_backend++) + { + LocalPgBackendStatus *local_beentry; + PgBackendStatus *beentry; + + /* Get the next one in the list */ + local_beentry = pgstat_get_local_beentry_by_index(curr_backend); + beentry = &local_beentry->backendStatus; + + /* Looking for specific PID, ignore all the others */ + if (beentry->st_procpid != pid) + continue; + + bktype = beentry->st_backendType; + break; + } + + /* Backend is gone */ + if (bktype == B_INVALID) + return (Datum) 0; + + bktype_desc = CStringGetTextDatum(GetBackendTypeDesc(bktype)); + bktype_stats = &backend_stats->stats; + reset_time = TimestampTzGetDatum(backend_stats->stat_reset_timestamp); + + /* + * In Assert builds, we can afford an extra loop through all of the + * counters checking that only expected stats are non-zero, since it keeps + * the non-Assert code cleaner. + */ + Assert(pgstat_bktype_io_stats_valid(bktype_stats, bktype)); + + for (int io_obj = 0; io_obj < IOOBJECT_NUM_TYPES; io_obj++) + { + const char *obj_name = pgstat_get_io_object_name(io_obj); + + for (int io_context = 0; io_context < IOCONTEXT_NUM_TYPES; io_context++) + { + const char *context_name = pgstat_get_io_context_name(io_context); + + Datum values[IO_NUM_COLUMNS] = {0}; + bool nulls[IO_NUM_COLUMNS] = {0}; + + /* + * Some combinations of BackendType, IOObject, and IOContext are + * not valid for any type of IOOp. In such cases, omit the entire + * row from the view. + */ + if (!pgstat_tracks_io_object(bktype, io_obj, io_context)) + continue; + + values[IO_COL_BACKEND_TYPE] = bktype_desc; + values[IO_COL_CONTEXT] = CStringGetTextDatum(context_name); + values[IO_COL_OBJECT] = CStringGetTextDatum(obj_name); + if (backend_stats->stat_reset_timestamp != 0) + values[IO_COL_RESET_TIME] = reset_time; + else + nulls[IO_COL_RESET_TIME] = true; + + /* + * Hard-code this to the value of BLCKSZ for now. Future values + * could include XLOG_BLCKSZ, once WAL IO is tracked, and constant + * multipliers, once non-block-oriented IO (e.g. temporary file + * IO) is tracked. + */ + values[IO_COL_CONVERSION] = Int64GetDatum(BLCKSZ); + + for (int io_op = 0; io_op < IOOP_NUM_TYPES; io_op++) + { + int op_idx = pgstat_get_io_op_index(io_op); + int time_idx = pgstat_get_io_time_index(io_op); + + /* + * Some combinations of BackendType and IOOp, of IOContext and + * IOOp, and of IOObject and IOOp are not tracked. Set these + * cells in the view NULL. + */ + if (pgstat_tracks_io_op(bktype, io_obj, io_context, io_op)) + { + PgStat_Counter count = + bktype_stats->counts[io_obj][io_context][io_op]; + + values[op_idx] = Int64GetDatum(count); + } + else + nulls[op_idx] = true; + + /* not every operation is timed */ + if (time_idx == IO_COL_INVALID) + continue; + + if (!nulls[op_idx]) + { + PgStat_Counter time = + bktype_stats->times[io_obj][io_context][io_op]; + + values[time_idx] = Float8GetDatum(pg_stat_us_to_ms(time)); + } + else + nulls[time_idx] = true; + } + + tuplestore_putvalues(rsinfo->setResult, rsinfo->setDesc, + values, nulls); + } + } + + return (Datum) 0; +} + /* * Returns statistics of WAL activity */ @@ -1779,6 +1927,27 @@ pg_stat_reset_single_function_counters(PG_FUNCTION_ARGS) PG_RETURN_VOID(); } +Datum +pg_stat_reset_backend_stats(PG_FUNCTION_ARGS) +{ + PGPROC *proc; + int backend_pid = PG_GETARG_INT32(0); + + proc = BackendPidGetProc(backend_pid); + + /* + * Could be an auxiliary process but would not report any stats due to + * pgstat_tracks_backend_bktype() anyway. So don't need an extra call to + * AuxiliaryPidGetProc(). + */ + if (!proc) + PG_RETURN_VOID(); + + pgstat_reset(PGSTAT_KIND_BACKEND, InvalidOid, GetNumberFromPGProc(proc)); + + PG_RETURN_VOID(); +} + /* Reset SLRU counters (a specific one or all of them). */ Datum pg_stat_reset_slru(PG_FUNCTION_ARGS) diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index 0f22c21723..437157ffa3 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -5913,6 +5913,15 @@ proargnames => '{backend_type,object,context,reads,read_time,writes,write_time,writebacks,writeback_time,extends,extend_time,op_bytes,hits,evictions,reuses,fsyncs,fsync_time,stats_reset}', prosrc => 'pg_stat_get_io' }, +{ oid => '8806', descr => 'statistics: backend IO statistics', + proname => 'pg_stat_get_backend_io', prorows => '5', proretset => 't', + provolatile => 'v', proparallel => 'r', prorettype => 'record', + proargtypes => 'int4', + proallargtypes => '{int4,text,text,text,int8,float8,int8,float8,int8,float8,int8,float8,int8,int8,int8,int8,int8,float8,timestamptz}', + proargmodes => '{i,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o}', + proargnames => '{backend_pid,backend_type,object,context,reads,read_time,writes,write_time,writebacks,writeback_time,extends,extend_time,op_bytes,hits,evictions,reuses,fsyncs,fsync_time,stats_reset}', + prosrc => 'pg_stat_get_backend_io' }, + { oid => '1136', descr => 'statistics: information about WAL activity', proname => 'pg_stat_get_wal', proisstrict => 'f', provolatile => 's', proparallel => 'r', prorettype => 'record', proargtypes => '', @@ -6052,6 +6061,11 @@ proname => 'pg_stat_reset_single_function_counters', provolatile => 'v', prorettype => 'void', proargtypes => 'oid', prosrc => 'pg_stat_reset_single_function_counters' }, +{ oid => '9987', + descr => 'statistics: reset statistics for a single backend', + proname => 'pg_stat_reset_backend_stats', provolatile => 'v', + prorettype => 'void', proargtypes => 'int4', + prosrc => 'pg_stat_reset_backend_stats' }, { oid => '2307', descr => 'statistics: reset collected statistics for a single SLRU', proname => 'pg_stat_reset_slru', proisstrict => 'f', provolatile => 'v', diff --git a/src/include/pgstat.h b/src/include/pgstat.h index ebfeef2f46..479773cfd2 100644 --- a/src/include/pgstat.h +++ b/src/include/pgstat.h @@ -49,14 +49,15 @@ #define PGSTAT_KIND_FUNCTION 3 /* per-function statistics */ #define PGSTAT_KIND_REPLSLOT 4 /* per-slot statistics */ #define PGSTAT_KIND_SUBSCRIPTION 5 /* per-subscription statistics */ +#define PGSTAT_KIND_BACKEND 6 /* per-backend statistics */ /* stats for fixed-numbered objects */ -#define PGSTAT_KIND_ARCHIVER 6 -#define PGSTAT_KIND_BGWRITER 7 -#define PGSTAT_KIND_CHECKPOINTER 8 -#define PGSTAT_KIND_IO 9 -#define PGSTAT_KIND_SLRU 10 -#define PGSTAT_KIND_WAL 11 +#define PGSTAT_KIND_ARCHIVER 7 +#define PGSTAT_KIND_BGWRITER 8 +#define PGSTAT_KIND_CHECKPOINTER 9 +#define PGSTAT_KIND_IO 10 +#define PGSTAT_KIND_SLRU 11 +#define PGSTAT_KIND_WAL 12 #define PGSTAT_KIND_BUILTIN_MIN PGSTAT_KIND_DATABASE #define PGSTAT_KIND_BUILTIN_MAX PGSTAT_KIND_WAL @@ -362,12 +363,23 @@ typedef struct PgStat_BktypeIO PgStat_Counter times[IOOBJECT_NUM_TYPES][IOCONTEXT_NUM_TYPES][IOOP_NUM_TYPES]; } PgStat_BktypeIO; +typedef struct PgStat_BackendPendingIO +{ + PgStat_Counter counts[IOOBJECT_NUM_TYPES][IOCONTEXT_NUM_TYPES][IOOP_NUM_TYPES]; + instr_time pending_times[IOOBJECT_NUM_TYPES][IOCONTEXT_NUM_TYPES][IOOP_NUM_TYPES]; +} PgStat_BackendPendingIO; + typedef struct PgStat_IO { TimestampTz stat_reset_timestamp; PgStat_BktypeIO stats[BACKEND_NUM_TYPES]; } PgStat_IO; +typedef struct PgStat_Backend +{ + TimestampTz stat_reset_timestamp; + PgStat_BktypeIO stats; +} PgStat_Backend; typedef struct PgStat_StatDBEntry { @@ -549,6 +561,13 @@ extern bool pgstat_have_entry(PgStat_Kind kind, Oid dboid, uint64 objid); extern void pgstat_report_archiver(const char *xlog, bool failed); extern PgStat_ArchiverStats *pgstat_fetch_stat_archiver(void); +/* + * Functions in pgstat_backend.c + */ + +extern PgStat_Backend *pgstat_fetch_proc_stat_io(ProcNumber procNumber); +extern bool pgstat_tracks_backend_bktype(BackendType bktype); +extern void pgstat_create_backend_stat(ProcNumber procnum); /* * Functions in pgstat_bgwriter.c diff --git a/src/include/utils/pgstat_internal.h b/src/include/utils/pgstat_internal.h index 7338bc1e28..811ed9b005 100644 --- a/src/include/utils/pgstat_internal.h +++ b/src/include/utils/pgstat_internal.h @@ -450,6 +450,11 @@ typedef struct PgStatShared_ReplSlot PgStat_StatReplSlotEntry stats; } PgStatShared_ReplSlot; +typedef struct PgStatShared_Backend +{ + PgStatShared_Common header; + PgStat_Backend stats; +} PgStatShared_Backend; /* * Central shared memory entry for the cumulative stats system. @@ -604,6 +609,15 @@ extern void pgstat_archiver_init_shmem_cb(void *stats); extern void pgstat_archiver_reset_all_cb(TimestampTz ts); extern void pgstat_archiver_snapshot_cb(void); +/* + * Functions in pgstat_backend.c + */ + +extern void pgstat_flush_backend(bool nowait); + +extern PgStat_BackendPendingIO *pgstat_prep_backend_pending(ProcNumber procnum); +extern bool pgstat_backend_flush_cb(PgStat_EntryRef *entry_ref, bool nowait); +extern void pgstat_backend_reset_timestamp_cb(PgStatShared_Common *header, TimestampTz ts); /* * Functions in pgstat_bgwriter.c diff --git a/src/test/regress/expected/stats.out b/src/test/regress/expected/stats.out index 56771f83ed..3447e7b75d 100644 --- a/src/test/regress/expected/stats.out +++ b/src/test/regress/expected/stats.out @@ -1249,7 +1249,7 @@ SELECT pg_stat_get_subscription_stats(NULL); (1 row) --- Test that the following operations are tracked in pg_stat_io: +-- Test that the following operations are tracked in pg_stat_io and in backend stats: -- - reads of target blocks into shared buffers -- - writes of shared buffers to permanent storage -- - extends of relations using shared buffers @@ -1259,11 +1259,19 @@ SELECT pg_stat_get_subscription_stats(NULL); -- be sure of the state of shared buffers at the point the test is run. -- Create a regular table and insert some data to generate IOCONTEXT_NORMAL -- extends. +SELECT pid AS checkpointer_pid FROM pg_stat_activity + WHERE backend_type = 'checkpointer' \gset SELECT sum(extends) AS io_sum_shared_before_extends FROM pg_stat_io WHERE context = 'normal' AND object = 'relation' \gset +SELECT sum(extends) AS my_io_sum_shared_before_extends + FROM pg_stat_get_backend_io(pg_backend_pid()) + WHERE context = 'normal' AND object = 'relation' \gset SELECT sum(writes) AS writes, sum(fsyncs) AS fsyncs FROM pg_stat_io WHERE object = 'relation' \gset io_sum_shared_before_ +SELECT sum(writes) AS writes, sum(fsyncs) AS fsyncs + FROM pg_stat_get_backend_io(pg_backend_pid()) + WHERE object = 'relation' \gset my_io_sum_shared_before_ CREATE TABLE test_io_shared(a int); INSERT INTO test_io_shared SELECT i FROM generate_series(1,100)i; SELECT pg_stat_force_next_flush(); @@ -1280,8 +1288,17 @@ SELECT :io_sum_shared_after_extends > :io_sum_shared_before_extends; t (1 row) +SELECT sum(extends) AS my_io_sum_shared_after_extends + FROM pg_stat_get_backend_io(pg_backend_pid()) + WHERE context = 'normal' AND object = 'relation' \gset +SELECT :my_io_sum_shared_after_extends > :my_io_sum_shared_before_extends; + ?column? +---------- + t +(1 row) + -- After a checkpoint, there should be some additional IOCONTEXT_NORMAL writes --- and fsyncs. +-- and fsyncs in the global stats (not for the backend). -- See comment above for rationale for two explicit CHECKPOINTs. CHECKPOINT; CHECKPOINT; @@ -1301,6 +1318,31 @@ SELECT current_setting('fsync') = 'off' t (1 row) +SELECT sum(writes) AS writes, sum(fsyncs) AS fsyncs + FROM pg_stat_get_backend_io(pg_backend_pid()) + WHERE object = 'relation' \gset my_io_sum_shared_after_ +SELECT :my_io_sum_shared_after_writes >= :my_io_sum_shared_before_writes; + ?column? +---------- + t +(1 row) + +SELECT current_setting('fsync') = 'off' + OR (:my_io_sum_shared_after_fsyncs = :my_io_sum_shared_before_fsyncs + AND :my_io_sum_shared_after_fsyncs= 0); + ?column? +---------- + t +(1 row) + +-- Don't return any rows if querying other backend's stats that are excluded +-- from the backend stats collection (like the checkpointer). +SELECT count(1) = 0 FROM pg_stat_get_backend_io(:checkpointer_pid); + ?column? +---------- + t +(1 row) + -- Change the tablespace so that the table is rewritten directly, then SELECT -- from it to cause it to be read back into shared buffers. SELECT sum(reads) AS io_sum_shared_before_reads @@ -1521,6 +1563,8 @@ SELECT pg_stat_have_stats('io', 0, 0); SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS io_stats_pre_reset FROM pg_stat_io \gset +SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS my_io_stats_pre_reset + FROM pg_stat_get_backend_io(pg_backend_pid()) \gset SELECT pg_stat_reset_shared('io'); pg_stat_reset_shared ---------------------- @@ -1535,6 +1579,30 @@ SELECT :io_stats_post_reset < :io_stats_pre_reset; t (1 row) +SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS my_io_stats_post_reset + FROM pg_stat_get_backend_io(pg_backend_pid()) \gset +-- pg_stat_reset_shared() did not reset backend IO stats +SELECT :my_io_stats_pre_reset <= :my_io_stats_post_reset; + ?column? +---------- + t +(1 row) + +-- but pg_stat_reset_backend_stats() does +SELECT pg_stat_reset_backend_stats(pg_backend_pid()); + pg_stat_reset_backend_stats +----------------------------- + +(1 row) + +SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS my_io_stats_post_backend_reset + FROM pg_stat_get_backend_io(pg_backend_pid()) \gset +SELECT :my_io_stats_pre_reset > :my_io_stats_post_backend_reset; + ?column? +---------- + t +(1 row) + -- test BRIN index doesn't block HOT update CREATE TABLE brin_hot ( id integer PRIMARY KEY, diff --git a/src/test/regress/sql/stats.sql b/src/test/regress/sql/stats.sql index 7147cc2f89..9c925005be 100644 --- a/src/test/regress/sql/stats.sql +++ b/src/test/regress/sql/stats.sql @@ -595,7 +595,7 @@ SELECT pg_stat_get_replication_slot(NULL); SELECT pg_stat_get_subscription_stats(NULL); --- Test that the following operations are tracked in pg_stat_io: +-- Test that the following operations are tracked in pg_stat_io and in backend stats: -- - reads of target blocks into shared buffers -- - writes of shared buffers to permanent storage -- - extends of relations using shared buffers @@ -607,20 +607,32 @@ SELECT pg_stat_get_subscription_stats(NULL); -- Create a regular table and insert some data to generate IOCONTEXT_NORMAL -- extends. +SELECT pid AS checkpointer_pid FROM pg_stat_activity + WHERE backend_type = 'checkpointer' \gset SELECT sum(extends) AS io_sum_shared_before_extends FROM pg_stat_io WHERE context = 'normal' AND object = 'relation' \gset +SELECT sum(extends) AS my_io_sum_shared_before_extends + FROM pg_stat_get_backend_io(pg_backend_pid()) + WHERE context = 'normal' AND object = 'relation' \gset SELECT sum(writes) AS writes, sum(fsyncs) AS fsyncs FROM pg_stat_io WHERE object = 'relation' \gset io_sum_shared_before_ +SELECT sum(writes) AS writes, sum(fsyncs) AS fsyncs + FROM pg_stat_get_backend_io(pg_backend_pid()) + WHERE object = 'relation' \gset my_io_sum_shared_before_ CREATE TABLE test_io_shared(a int); INSERT INTO test_io_shared SELECT i FROM generate_series(1,100)i; SELECT pg_stat_force_next_flush(); SELECT sum(extends) AS io_sum_shared_after_extends FROM pg_stat_io WHERE context = 'normal' AND object = 'relation' \gset SELECT :io_sum_shared_after_extends > :io_sum_shared_before_extends; +SELECT sum(extends) AS my_io_sum_shared_after_extends + FROM pg_stat_get_backend_io(pg_backend_pid()) + WHERE context = 'normal' AND object = 'relation' \gset +SELECT :my_io_sum_shared_after_extends > :my_io_sum_shared_before_extends; -- After a checkpoint, there should be some additional IOCONTEXT_NORMAL writes --- and fsyncs. +-- and fsyncs in the global stats (not for the backend). -- See comment above for rationale for two explicit CHECKPOINTs. CHECKPOINT; CHECKPOINT; @@ -630,6 +642,17 @@ SELECT sum(writes) AS writes, sum(fsyncs) AS fsyncs SELECT :io_sum_shared_after_writes > :io_sum_shared_before_writes; SELECT current_setting('fsync') = 'off' OR :io_sum_shared_after_fsyncs > :io_sum_shared_before_fsyncs; +SELECT sum(writes) AS writes, sum(fsyncs) AS fsyncs + FROM pg_stat_get_backend_io(pg_backend_pid()) + WHERE object = 'relation' \gset my_io_sum_shared_after_ +SELECT :my_io_sum_shared_after_writes >= :my_io_sum_shared_before_writes; +SELECT current_setting('fsync') = 'off' + OR (:my_io_sum_shared_after_fsyncs = :my_io_sum_shared_before_fsyncs + AND :my_io_sum_shared_after_fsyncs= 0); + +-- Don't return any rows if querying other backend's stats that are excluded +-- from the backend stats collection (like the checkpointer). +SELECT count(1) = 0 FROM pg_stat_get_backend_io(:checkpointer_pid); -- Change the tablespace so that the table is rewritten directly, then SELECT -- from it to cause it to be read back into shared buffers. @@ -762,10 +785,21 @@ SELECT :io_sum_bulkwrite_strategy_extends_after > :io_sum_bulkwrite_strategy_ext SELECT pg_stat_have_stats('io', 0, 0); SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS io_stats_pre_reset FROM pg_stat_io \gset +SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS my_io_stats_pre_reset + FROM pg_stat_get_backend_io(pg_backend_pid()) \gset SELECT pg_stat_reset_shared('io'); SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS io_stats_post_reset FROM pg_stat_io \gset SELECT :io_stats_post_reset < :io_stats_pre_reset; +SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS my_io_stats_post_reset + FROM pg_stat_get_backend_io(pg_backend_pid()) \gset +-- pg_stat_reset_shared() did not reset backend IO stats +SELECT :my_io_stats_pre_reset <= :my_io_stats_post_reset; +-- but pg_stat_reset_backend_stats() does +SELECT pg_stat_reset_backend_stats(pg_backend_pid()); +SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS my_io_stats_post_backend_reset + FROM pg_stat_get_backend_io(pg_backend_pid()) \gset +SELECT :my_io_stats_pre_reset > :my_io_stats_post_backend_reset; -- test BRIN index doesn't block HOT update diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index ce33e55bf1..398dd92527 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2121,6 +2121,7 @@ PgFdwSamplingMethod PgFdwScanState PgIfAddrCallback PgStatShared_Archiver +PgStatShared_Backend PgStatShared_BgWriter PgStatShared_Checkpointer PgStatShared_Common @@ -2136,6 +2137,8 @@ PgStatShared_SLRU PgStatShared_Subscription PgStatShared_Wal PgStat_ArchiverStats +PgStat_Backend +PgStat_BackendPendingIO PgStat_BackendSubEntry PgStat_BgWriterStats PgStat_BktypeIO -- 2.34.1 --fWBP7ZYU669/m9FI-- ^ permalink raw reply [nested|flat] 25+ messages in thread
* [PATCH v5 1/4] per backend I/O statistics @ 2024-10-28 12:50 Bertrand Drouvot <[email protected]> 0 siblings, 0 replies; 25+ messages in thread From: Bertrand Drouvot @ 2024-10-28 12:50 UTC (permalink / raw) While pg_stat_io provides cluster-wide I/O statistics, this commit adds a new statistics kind and a new pg_my_stat_io view to display "my" backend I/O statistics. The new KIND is named PGSTAT_KIND_PER_BACKEND as it could be used in the future to store other statistics (than the I/O ones) per backend. The new KIND is a variable-numbered one and has an automatic cap on the maximum number of entries (as its hash key contains the proc number). There is no need to serialize the per backend I/O stats to disk (no point to see stats for backends that do not exist anymore after a re-start), so a new "to_serialize" field is added in the PgStat_KindInfo struct. Also adding a new function pg_stat_reset_single_backend_io_counters() to be able to reset the I/O stats for a given backend pid. A subsequent commit will add a new pg_stat_get_backend_io() function to be able to retrieve the I/O statistics for a given backend pid. XXX: Bump catalog version needs to be done. --- doc/src/sgml/config.sgml | 4 +- doc/src/sgml/monitoring.sgml | 43 ++++++ src/backend/catalog/system_functions.sql | 2 + src/backend/catalog/system_views.sql | 22 ++++ src/backend/utils/activity/backend_status.c | 3 + src/backend/utils/activity/pgstat.c | 34 ++++- src/backend/utils/activity/pgstat_io.c | 114 ++++++++++++++-- src/backend/utils/activity/pgstat_shmem.c | 30 ++++- src/backend/utils/adt/pgstatfuncs.c | 124 ++++++++++++++++++ src/include/catalog/pg_proc.dat | 14 ++ src/include/pgstat.h | 27 +++- src/include/utils/pgstat_internal.h | 13 ++ .../injection_points/injection_stats.c | 1 + src/test/regress/expected/rules.out | 19 +++ src/test/regress/expected/stats.out | 60 ++++++++- src/test/regress/sql/stats.sql | 31 ++++- src/tools/pgindent/typedefs.list | 2 + 17 files changed, 517 insertions(+), 26 deletions(-) 10.2% doc/src/sgml/ 29.7% src/backend/utils/activity/ 19.2% src/backend/utils/adt/ 5.1% src/include/catalog/ 7.2% src/include/ 14.7% src/test/regress/expected/ 10.4% src/test/regress/sql/ 3.1% src/ diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index d54f904956..fbabf371a6 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -8361,7 +8361,9 @@ COPY postgres_log FROM '/full/path/to/logfile.csv' WITH csv; displayed in <link linkend="monitoring-pg-stat-database-view"> <structname>pg_stat_database</structname></link>, <link linkend="monitoring-pg-stat-io-view"> - <structname>pg_stat_io</structname></link>, in the output of + <structname>pg_stat_io</structname></link>, + <link linkend="monitoring-pg-my-stat-io-view"> + <structname>pg_my_stat_io</structname></link>, in the output of <xref linkend="sql-explain"/> when the <literal>BUFFERS</literal> option is used, in the output of <xref linkend="sql-vacuum"/> when the <literal>VERBOSE</literal> option is used, by autovacuum diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml index 331315f8d3..fc6aded3da 100644 --- a/doc/src/sgml/monitoring.sgml +++ b/doc/src/sgml/monitoring.sgml @@ -488,6 +488,16 @@ postgres 27093 0.0 0.0 30096 2752 ? Ss 11:34 0:00 postgres: ser </entry> </row> + <row> + <entry><structname>pg_my_stat_io</structname><indexterm><primary>pg_my_stat_io</primary></indexterm></entry> + <entry> + One row for each combination of context and target object containing + my backend I/O statistics. + See <link linkend="monitoring-pg-my-stat-io-view"> + <structname>pg_my_stat_io</structname></link> for details. + </entry> + </row> + <row> <entry><structname>pg_stat_replication_slots</structname><indexterm><primary>pg_stat_replication_slots</primary></indexterm></entry> <entry>One row per replication slot, showing statistics about the @@ -2946,7 +2956,23 @@ description | Waiting for a newly initialized WAL file to reach durable storage </para> </note> + </sect2> + <sect2 id="monitoring-pg-my-stat-io-view"> + <title><structname>pg_my_stat_io</structname></title> + + <indexterm> + <primary>pg_my_stat_io</primary> + </indexterm> + + <para> + The <structname>pg_my_stat_io</structname> view will contain one row for each + combination of target I/O object and I/O context, showing + my backend I/O statistics. Combinations which do not make sense are + omitted. The fields are exactly the same as the ones in the <link + linkend="monitoring-pg-stat-io-view"> <structname>pg_stat_io</structname></link> + view. + </para> </sect2> @@ -4971,6 +4997,23 @@ description | Waiting for a newly initialized WAL file to reach durable storage </para></entry> </row> + <row> + <entry role="func_table_entry"><para role="func_signature"> + <indexterm> + <primary>pg_stat_reset_single_backend_io_counters</primary> + </indexterm> + <function>pg_stat_reset_single_backend_io_counters</function> ( <type>int4</type> ) + <returnvalue>void</returnvalue> + </para> + <para> + Resets I/O statistics for a single backend to zero. + </para> + <para> + This function is restricted to superusers by default, but other users + can be granted EXECUTE to run the function. + </para></entry> + </row> + <row> <entry role="func_table_entry"><para role="func_signature"> <indexterm> diff --git a/src/backend/catalog/system_functions.sql b/src/backend/catalog/system_functions.sql index c51dfca802..8a1ae44fca 100644 --- a/src/backend/catalog/system_functions.sql +++ b/src/backend/catalog/system_functions.sql @@ -711,6 +711,8 @@ REVOKE EXECUTE ON FUNCTION pg_stat_reset_single_table_counters(oid) FROM public; REVOKE EXECUTE ON FUNCTION pg_stat_reset_single_function_counters(oid) FROM public; +REVOKE EXECUTE ON FUNCTION pg_stat_reset_single_backend_io_counters(int4) FROM public; + REVOKE EXECUTE ON FUNCTION pg_stat_reset_replication_slot(text) FROM public; REVOKE EXECUTE ON FUNCTION pg_stat_have_stats(text, oid, int8) FROM public; diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index 3456b821bc..09af4a40a8 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1170,6 +1170,28 @@ SELECT b.stats_reset FROM pg_stat_get_io() b; +CREATE VIEW pg_my_stat_io AS +SELECT + b.backend_type, + b.object, + b.context, + b.reads, + b.read_time, + b.writes, + b.write_time, + b.writebacks, + b.writeback_time, + b.extends, + b.extend_time, + b.op_bytes, + b.hits, + b.evictions, + b.reuses, + b.fsyncs, + b.fsync_time, + b.stats_reset +FROM pg_stat_get_my_io() b; + CREATE VIEW pg_stat_wal AS SELECT w.wal_records, diff --git a/src/backend/utils/activity/backend_status.c b/src/backend/utils/activity/backend_status.c index bdb3a296ca..aa683e150c 100644 --- a/src/backend/utils/activity/backend_status.c +++ b/src/backend/utils/activity/backend_status.c @@ -249,6 +249,9 @@ pgstat_beinit(void) Assert(MyProcNumber >= 0 && MyProcNumber < NumBackendStatSlots); MyBEEntry = &BackendStatusArray[MyProcNumber]; + /* Create the per backend stat entry */ + pgstat_create_backend_stat(MyProcNumber); + /* Set up a process-exit hook to clean up */ on_shmem_exit(pgstat_beshutdown_hook, 0); } diff --git a/src/backend/utils/activity/pgstat.c b/src/backend/utils/activity/pgstat.c index ea8c5691e8..aacf61c9a4 100644 --- a/src/backend/utils/activity/pgstat.c +++ b/src/backend/utils/activity/pgstat.c @@ -281,6 +281,7 @@ static const PgStat_KindInfo pgstat_kind_builtin_infos[PGSTAT_KIND_BUILTIN_SIZE] .name = "database", .fixed_amount = false, + .to_serialize = true, /* so pg_stat_database entries can be seen in all databases */ .accessed_across_databases = true, @@ -297,6 +298,7 @@ static const PgStat_KindInfo pgstat_kind_builtin_infos[PGSTAT_KIND_BUILTIN_SIZE] .name = "relation", .fixed_amount = false, + .to_serialize = true, .shared_size = sizeof(PgStatShared_Relation), .shared_data_off = offsetof(PgStatShared_Relation, stats), @@ -311,6 +313,7 @@ static const PgStat_KindInfo pgstat_kind_builtin_infos[PGSTAT_KIND_BUILTIN_SIZE] .name = "function", .fixed_amount = false, + .to_serialize = true, .shared_size = sizeof(PgStatShared_Function), .shared_data_off = offsetof(PgStatShared_Function, stats), @@ -324,6 +327,7 @@ static const PgStat_KindInfo pgstat_kind_builtin_infos[PGSTAT_KIND_BUILTIN_SIZE] .name = "replslot", .fixed_amount = false, + .to_serialize = true, .accessed_across_databases = true, @@ -340,6 +344,7 @@ static const PgStat_KindInfo pgstat_kind_builtin_infos[PGSTAT_KIND_BUILTIN_SIZE] .name = "subscription", .fixed_amount = false, + .to_serialize = true, /* so pg_stat_subscription_stats entries can be seen in all databases */ .accessed_across_databases = true, @@ -352,6 +357,22 @@ static const PgStat_KindInfo pgstat_kind_builtin_infos[PGSTAT_KIND_BUILTIN_SIZE] .reset_timestamp_cb = pgstat_subscription_reset_timestamp_cb, }, + [PGSTAT_KIND_PER_BACKEND] = { + .name = "perbackend", + + .fixed_amount = false, + .to_serialize = false, + + .accessed_across_databases = true, + + .shared_size = sizeof(PgStatShared_Backend), + .shared_data_off = offsetof(PgStatShared_Backend, stats), + .shared_data_len = sizeof(((PgStatShared_Backend *) 0)->stats), + .pending_size = sizeof(PgStat_PendingIO), + + .flush_pending_cb = pgstat_per_backend_flush_cb, + .reset_timestamp_cb = pgstat_backend_reset_timestamp_cb, + }, /* stats for fixed-numbered (mostly 1) objects */ @@ -359,6 +380,7 @@ static const PgStat_KindInfo pgstat_kind_builtin_infos[PGSTAT_KIND_BUILTIN_SIZE] .name = "archiver", .fixed_amount = true, + .to_serialize = true, .snapshot_ctl_off = offsetof(PgStat_Snapshot, archiver), .shared_ctl_off = offsetof(PgStat_ShmemControl, archiver), @@ -374,6 +396,7 @@ static const PgStat_KindInfo pgstat_kind_builtin_infos[PGSTAT_KIND_BUILTIN_SIZE] .name = "bgwriter", .fixed_amount = true, + .to_serialize = true, .snapshot_ctl_off = offsetof(PgStat_Snapshot, bgwriter), .shared_ctl_off = offsetof(PgStat_ShmemControl, bgwriter), @@ -389,6 +412,7 @@ static const PgStat_KindInfo pgstat_kind_builtin_infos[PGSTAT_KIND_BUILTIN_SIZE] .name = "checkpointer", .fixed_amount = true, + .to_serialize = true, .snapshot_ctl_off = offsetof(PgStat_Snapshot, checkpointer), .shared_ctl_off = offsetof(PgStat_ShmemControl, checkpointer), @@ -404,6 +428,7 @@ static const PgStat_KindInfo pgstat_kind_builtin_infos[PGSTAT_KIND_BUILTIN_SIZE] .name = "io", .fixed_amount = true, + .to_serialize = true, .snapshot_ctl_off = offsetof(PgStat_Snapshot, io), .shared_ctl_off = offsetof(PgStat_ShmemControl, io), @@ -421,6 +446,7 @@ static const PgStat_KindInfo pgstat_kind_builtin_infos[PGSTAT_KIND_BUILTIN_SIZE] .name = "slru", .fixed_amount = true, + .to_serialize = true, .snapshot_ctl_off = offsetof(PgStat_Snapshot, slru), .shared_ctl_off = offsetof(PgStat_ShmemControl, slru), @@ -438,6 +464,7 @@ static const PgStat_KindInfo pgstat_kind_builtin_infos[PGSTAT_KIND_BUILTIN_SIZE] .name = "wal", .fixed_amount = true, + .to_serialize = true, .snapshot_ctl_off = offsetof(PgStat_Snapshot, wal), .shared_ctl_off = offsetof(PgStat_ShmemControl, wal), @@ -515,7 +542,7 @@ pgstat_discard_stats(void) /* * Reset stats contents. This will set reset timestamps of fixed-numbered - * stats to the current time (no variable stats exist). + * stats to the current time (the per-backend variable stat exists too). */ pgstat_reset_after_failure(); } @@ -756,7 +783,7 @@ pgstat_report_stat(bool force) partial_flush = false; - /* flush database / relation / function / ... stats */ + /* flush database / relation / function / backend / ... stats */ partial_flush |= pgstat_flush_pending_entries(nowait); /* flush of fixed-numbered stats */ @@ -1660,6 +1687,9 @@ pgstat_write_statsfile(XLogRecPtr redo) kind_info = pgstat_get_kind_info(ps->key.kind); + if (!kind_info->to_serialize) + continue; + /* if not dropped the valid-entry refcount should exist */ Assert(pg_atomic_read_u32(&ps->refcount) > 0); diff --git a/src/backend/utils/activity/pgstat_io.c b/src/backend/utils/activity/pgstat_io.c index cc2ffc78aa..a2bb11adc9 100644 --- a/src/backend/utils/activity/pgstat_io.c +++ b/src/backend/utils/activity/pgstat_io.c @@ -21,13 +21,6 @@ #include "utils/pgstat_internal.h" -typedef struct PgStat_PendingIO -{ - PgStat_Counter counts[IOOBJECT_NUM_TYPES][IOCONTEXT_NUM_TYPES][IOOP_NUM_TYPES]; - instr_time pending_times[IOOBJECT_NUM_TYPES][IOCONTEXT_NUM_TYPES][IOOP_NUM_TYPES]; -} PgStat_PendingIO; - - static PgStat_PendingIO PendingIOStats; static bool have_iostats = false; @@ -82,12 +75,17 @@ pgstat_count_io_op(IOObject io_object, IOContext io_context, IOOp io_op) void pgstat_count_io_op_n(IOObject io_object, IOContext io_context, IOOp io_op, uint32 cnt) { + PgStat_PendingIO *entry_ref; + Assert((unsigned int) io_object < IOOBJECT_NUM_TYPES); Assert((unsigned int) io_context < IOCONTEXT_NUM_TYPES); Assert((unsigned int) io_op < IOOP_NUM_TYPES); Assert(pgstat_tracks_io_op(MyBackendType, io_object, io_context, io_op)); + entry_ref = pgstat_prep_per_backend_pending(MyProcNumber); + PendingIOStats.counts[io_object][io_context][io_op] += cnt; + entry_ref->counts[io_object][io_context][io_op] += cnt; have_iostats = true; } @@ -122,6 +120,10 @@ void pgstat_count_io_op_time(IOObject io_object, IOContext io_context, IOOp io_op, instr_time start_time, uint32 cnt) { + PgStat_PendingIO *entry_ref; + + entry_ref = pgstat_prep_per_backend_pending(MyProcNumber); + if (track_io_timing) { instr_time io_time; @@ -148,6 +150,8 @@ pgstat_count_io_op_time(IOObject io_object, IOContext io_context, IOOp io_op, INSTR_TIME_ADD(PendingIOStats.pending_times[io_object][io_context][io_op], io_time); + INSTR_TIME_ADD(entry_ref->pending_times[io_object][io_context][io_op], + io_time); } pgstat_count_io_op_n(io_object, io_context, io_op, cnt); @@ -161,6 +165,13 @@ pgstat_fetch_stat_io(void) return &pgStatLocal.snapshot.io; } +PgStat_Backend * +pgstat_fetch_my_stat_io(void) +{ + return (PgStat_Backend *) + pgstat_fetch_entry(PGSTAT_KIND_PER_BACKEND, InvalidOid, MyProcNumber); +} + /* * Check if there any IO stats waiting for flush. */ @@ -171,12 +182,18 @@ pgstat_io_have_pending_cb(void) } /* - * Simpler wrapper of pgstat_io_flush_cb() + * Simpler wrapper of pgstat_io_flush_cb() and pgstat_per_backend_flush_cb(). */ void pgstat_flush_io(bool nowait) { + PgStat_EntryRef *entry_ref; + + entry_ref = pgstat_get_entry_ref(PGSTAT_KIND_PER_BACKEND, InvalidOid, + MyProcNumber, false, NULL); + (void) pgstat_io_flush_cb(nowait); + (void) pgstat_per_backend_flush_cb(entry_ref, nowait); } /* @@ -235,6 +252,49 @@ pgstat_io_flush_cb(bool nowait) return false; } +/* + * Flush out locally pending backend statistics + * + * If no stats have been recorded, this function returns false. + */ +bool +pgstat_per_backend_flush_cb(PgStat_EntryRef *entry_ref, bool nowait) +{ + PgStatShared_Backend *shbackendioent; + PgStat_PendingIO *pendingent; + PgStat_BktypeIO *bktype_shstats; + + if (!pgstat_lock_entry(entry_ref, nowait)) + return false; + + shbackendioent = (PgStatShared_Backend *) entry_ref->shared_stats; + bktype_shstats = &shbackendioent->stats.stats; + pendingent = (PgStat_PendingIO *) entry_ref->pending; + + for (int io_object = 0; io_object < IOOBJECT_NUM_TYPES; io_object++) + { + for (int io_context = 0; io_context < IOCONTEXT_NUM_TYPES; io_context++) + { + for (int io_op = 0; io_op < IOOP_NUM_TYPES; io_op++) + { + instr_time time; + + bktype_shstats->counts[io_object][io_context][io_op] += + pendingent->counts[io_object][io_context][io_op]; + + time = pendingent->pending_times[io_object][io_context][io_op]; + + bktype_shstats->times[io_object][io_context][io_op] += + INSTR_TIME_GET_MICROSEC(time); + } + } + } + + pgstat_unlock_entry(entry_ref); + + return true; +} + const char * pgstat_get_io_context_name(IOContext io_context) { @@ -325,6 +385,38 @@ pgstat_io_snapshot_cb(void) } } +void +pgstat_create_backend_stat(ProcNumber procnum) +{ + PgStat_EntryRef *entry_ref; + PgStatShared_Backend *shstatent; + + entry_ref = pgstat_prep_pending_entry(PGSTAT_KIND_PER_BACKEND, InvalidOid, + procnum, NULL); + + shstatent = (PgStatShared_Backend *) entry_ref->shared_stats; + + /* + * NB: need to accept that there might be stats from an older backend, + * e.g. if we previously used this proc number. + */ + memset(&shstatent->stats, 0, sizeof(shstatent->stats)); + + /* set the backend type */ + shstatent->stats.bktype = MyBackendType; +} + +PgStat_PendingIO * +pgstat_prep_per_backend_pending(ProcNumber procnum) +{ + PgStat_EntryRef *entry_ref; + + entry_ref = pgstat_prep_pending_entry(PGSTAT_KIND_PER_BACKEND, InvalidOid, + procnum, NULL); + + return entry_ref->pending; +} + /* * IO statistics are not collected for all BackendTypes. * @@ -503,3 +595,9 @@ pgstat_tracks_io_op(BackendType bktype, IOObject io_object, return true; } + +void +pgstat_backend_reset_timestamp_cb(PgStatShared_Common *header, TimestampTz ts) +{ + ((PgStatShared_Backend *) header)->stats.stat_reset_timestamp = ts; +} diff --git a/src/backend/utils/activity/pgstat_shmem.c b/src/backend/utils/activity/pgstat_shmem.c index c1b7ff76b1..b871c1d7d0 100644 --- a/src/backend/utils/activity/pgstat_shmem.c +++ b/src/backend/utils/activity/pgstat_shmem.c @@ -813,13 +813,24 @@ pgstat_free_entry(PgStatShared_HashEntry *shent, dshash_seq_status *hstat) */ static bool pgstat_drop_entry_internal(PgStatShared_HashEntry *shent, - dshash_seq_status *hstat) + dshash_seq_status *hstat, bool ignore_aux_proc) { Assert(shent->body != InvalidDsaPointer); /* should already have released local reference */ if (pgStatEntryRefHash) - Assert(!pgstat_entry_ref_hash_lookup(pgStatEntryRefHash, shent->key)); + { + /* + * The following assertion is not correct when coming from + * pgstat_discard_stats() and for per backend stats linked to an + * auxiliary process. + */ + if (ignore_aux_proc && shent->key.kind == PGSTAT_KIND_PER_BACKEND && + MyProcNumber >= MaxBackends) + return false; + else + Assert(!pgstat_entry_ref_hash_lookup(pgStatEntryRefHash, shent->key)); + } /* * Signal that the entry is dropped - this will eventually cause other @@ -882,7 +893,7 @@ pgstat_drop_database_and_contents(Oid dboid) if (p->key.dboid != dboid) continue; - if (!pgstat_drop_entry_internal(p, &hstat)) + if (!pgstat_drop_entry_internal(p, &hstat, false)) { /* * Even statistics for a dropped database might currently be @@ -941,7 +952,7 @@ pgstat_drop_entry(PgStat_Kind kind, Oid dboid, uint64 objid) shent = dshash_find(pgStatLocal.shared_hash, &key, true); if (shent) { - freed = pgstat_drop_entry_internal(shent, NULL); + freed = pgstat_drop_entry_internal(shent, NULL, false); /* * Database stats contain other stats. Drop those as well when @@ -969,7 +980,7 @@ pgstat_drop_all_entries(void) if (ps->dropped) continue; - if (!pgstat_drop_entry_internal(ps, &hstat)) + if (!pgstat_drop_entry_internal(ps, &hstat, true)) not_freed_count++; } dshash_seq_term(&hstat); @@ -982,11 +993,20 @@ static void shared_stat_reset_contents(PgStat_Kind kind, PgStatShared_Common *header, TimestampTz ts) { + BackendType bktype; const PgStat_KindInfo *kind_info = pgstat_get_kind_info(kind); + /* save the bktype */ + if (kind == PGSTAT_KIND_PER_BACKEND) + bktype = ((PgStatShared_Backend *) header)->stats.bktype; + memset(pgstat_get_entry_data(kind, header), 0, pgstat_get_entry_len(kind)); + /* restore the bktype */ + if (kind == PGSTAT_KIND_PER_BACKEND) + ((PgStatShared_Backend *) header)->stats.bktype = bktype; + if (kind_info->reset_timestamp_cb) kind_info->reset_timestamp_cb(header, ts); } diff --git a/src/backend/utils/adt/pgstatfuncs.c b/src/backend/utils/adt/pgstatfuncs.c index f7b50e0b5a..fd16986156 100644 --- a/src/backend/utils/adt/pgstatfuncs.c +++ b/src/backend/utils/adt/pgstatfuncs.c @@ -1474,6 +1474,111 @@ pg_stat_get_io(PG_FUNCTION_ARGS) return (Datum) 0; } +Datum +pg_stat_get_my_io(PG_FUNCTION_ARGS) +{ + ReturnSetInfo *rsinfo; + PgStat_Backend *backend_stats; + Datum bktype_desc; + PgStat_BktypeIO *bktype_stats; + BackendType bktype; + Datum reset_time; + + InitMaterializedSRF(fcinfo, 0); + rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; + + backend_stats = pgstat_fetch_my_stat_io(); + + bktype = backend_stats->bktype; + bktype_desc = CStringGetTextDatum(GetBackendTypeDesc(bktype)); + bktype_stats = &backend_stats->stats; + reset_time = TimestampTzGetDatum(backend_stats->stat_reset_timestamp); + + /* + * In Assert builds, we can afford an extra loop through all of the + * counters checking that only expected stats are non-zero, since it keeps + * the non-Assert code cleaner. + */ + Assert(pgstat_bktype_io_stats_valid(bktype_stats, bktype)); + + for (int io_obj = 0; io_obj < IOOBJECT_NUM_TYPES; io_obj++) + { + const char *obj_name = pgstat_get_io_object_name(io_obj); + + for (int io_context = 0; io_context < IOCONTEXT_NUM_TYPES; io_context++) + { + const char *context_name = pgstat_get_io_context_name(io_context); + + Datum values[IO_NUM_COLUMNS] = {0}; + bool nulls[IO_NUM_COLUMNS] = {0}; + + /* + * Some combinations of BackendType, IOObject, and IOContext are + * not valid for any type of IOOp. In such cases, omit the entire + * row from the view. + */ + if (!pgstat_tracks_io_object(bktype, io_obj, io_context)) + continue; + + values[IO_COL_BACKEND_TYPE] = bktype_desc; + values[IO_COL_CONTEXT] = CStringGetTextDatum(context_name); + values[IO_COL_OBJECT] = CStringGetTextDatum(obj_name); + if (backend_stats->stat_reset_timestamp != 0) + values[IO_COL_RESET_TIME] = reset_time; + else + nulls[IO_COL_RESET_TIME] = true; + + /* + * Hard-code this to the value of BLCKSZ for now. Future values + * could include XLOG_BLCKSZ, once WAL IO is tracked, and constant + * multipliers, once non-block-oriented IO (e.g. temporary file + * IO) is tracked. + */ + values[IO_COL_CONVERSION] = Int64GetDatum(BLCKSZ); + + for (int io_op = 0; io_op < IOOP_NUM_TYPES; io_op++) + { + int op_idx = pgstat_get_io_op_index(io_op); + int time_idx = pgstat_get_io_time_index(io_op); + + /* + * Some combinations of BackendType and IOOp, of IOContext and + * IOOp, and of IOObject and IOOp are not tracked. Set these + * cells in the view NULL. + */ + if (pgstat_tracks_io_op(bktype, io_obj, io_context, io_op)) + { + PgStat_Counter count = + bktype_stats->counts[io_obj][io_context][io_op]; + + values[op_idx] = Int64GetDatum(count); + } + else + nulls[op_idx] = true; + + /* not every operation is timed */ + if (time_idx == IO_COL_INVALID) + continue; + + if (!nulls[op_idx]) + { + PgStat_Counter time = + bktype_stats->times[io_obj][io_context][io_op]; + + values[time_idx] = Float8GetDatum(pg_stat_us_to_ms(time)); + } + else + nulls[time_idx] = true; + } + + tuplestore_putvalues(rsinfo->setResult, rsinfo->setDesc, + values, nulls); + } + } + + return (Datum) 0; +} + /* * Returns statistics of WAL activity */ @@ -1722,6 +1827,7 @@ pg_stat_reset_shared(PG_FUNCTION_ARGS) pgstat_reset_of_kind(PGSTAT_KIND_BGWRITER); pgstat_reset_of_kind(PGSTAT_KIND_CHECKPOINTER); pgstat_reset_of_kind(PGSTAT_KIND_IO); + pgstat_reset_of_kind(PGSTAT_KIND_PER_BACKEND); XLogPrefetchResetStats(); pgstat_reset_of_kind(PGSTAT_KIND_SLRU); pgstat_reset_of_kind(PGSTAT_KIND_WAL); @@ -1779,6 +1885,24 @@ pg_stat_reset_single_function_counters(PG_FUNCTION_ARGS) PG_RETURN_VOID(); } +Datum +pg_stat_reset_single_backend_io_counters(PG_FUNCTION_ARGS) +{ + PGPROC *proc; + int backend_pid = PG_GETARG_INT32(0); + + proc = BackendPidGetProc(backend_pid); + + /* Maybe an auxiliary process? */ + if (proc == NULL) + proc = AuxiliaryPidGetProc(backend_pid); + + if (proc) + pgstat_reset(PGSTAT_KIND_PER_BACKEND, InvalidOid, GetNumberFromPGProc(proc)); + + PG_RETURN_VOID(); +} + /* Reset SLRU counters (a specific one or all of them). */ Datum pg_stat_reset_slru(PG_FUNCTION_ARGS) diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index f23321a41f..89eb89efe4 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -5903,6 +5903,15 @@ proargnames => '{backend_type,object,context,reads,read_time,writes,write_time,writebacks,writeback_time,extends,extend_time,op_bytes,hits,evictions,reuses,fsyncs,fsync_time,stats_reset}', prosrc => 'pg_stat_get_io' }, +{ oid => '8806', descr => 'statistics: my backend IO statistics', + proname => 'pg_stat_get_my_io', prorows => '5', proretset => 't', + provolatile => 'v', proparallel => 'r', prorettype => 'record', + proargtypes => '', + proallargtypes => '{text,text,text,int8,float8,int8,float8,int8,float8,int8,float8,int8,int8,int8,int8,int8,float8,timestamptz}', + proargmodes => '{o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o}', + proargnames => '{backend_type,object,context,reads,read_time,writes,write_time,writebacks,writeback_time,extends,extend_time,op_bytes,hits,evictions,reuses,fsyncs,fsync_time,stats_reset}', + prosrc => 'pg_stat_get_my_io' }, + { oid => '1136', descr => 'statistics: information about WAL activity', proname => 'pg_stat_get_wal', proisstrict => 'f', provolatile => 's', proparallel => 'r', prorettype => 'record', proargtypes => '', @@ -6042,6 +6051,11 @@ proname => 'pg_stat_reset_single_function_counters', provolatile => 'v', prorettype => 'void', proargtypes => 'oid', prosrc => 'pg_stat_reset_single_function_counters' }, +{ oid => '9987', + descr => 'statistics: reset collected IO statistics for a single backend', + proname => 'pg_stat_reset_single_backend_io_counters', provolatile => 'v', + prorettype => 'void', proargtypes => 'int4', + prosrc => 'pg_stat_reset_single_backend_io_counters' }, { oid => '2307', descr => 'statistics: reset collected statistics for a single SLRU', proname => 'pg_stat_reset_slru', proisstrict => 'f', provolatile => 'v', diff --git a/src/include/pgstat.h b/src/include/pgstat.h index df53fa2d4f..13b9c71759 100644 --- a/src/include/pgstat.h +++ b/src/include/pgstat.h @@ -49,14 +49,15 @@ #define PGSTAT_KIND_FUNCTION 3 /* per-function statistics */ #define PGSTAT_KIND_REPLSLOT 4 /* per-slot statistics */ #define PGSTAT_KIND_SUBSCRIPTION 5 /* per-subscription statistics */ +#define PGSTAT_KIND_PER_BACKEND 6 /* stats for fixed-numbered objects */ -#define PGSTAT_KIND_ARCHIVER 6 -#define PGSTAT_KIND_BGWRITER 7 -#define PGSTAT_KIND_CHECKPOINTER 8 -#define PGSTAT_KIND_IO 9 -#define PGSTAT_KIND_SLRU 10 -#define PGSTAT_KIND_WAL 11 +#define PGSTAT_KIND_ARCHIVER 7 +#define PGSTAT_KIND_BGWRITER 8 +#define PGSTAT_KIND_CHECKPOINTER 9 +#define PGSTAT_KIND_IO 10 +#define PGSTAT_KIND_SLRU 11 +#define PGSTAT_KIND_WAL 12 #define PGSTAT_KIND_BUILTIN_MIN PGSTAT_KIND_DATABASE #define PGSTAT_KIND_BUILTIN_MAX PGSTAT_KIND_WAL @@ -347,12 +348,24 @@ typedef struct PgStat_BktypeIO PgStat_Counter times[IOOBJECT_NUM_TYPES][IOCONTEXT_NUM_TYPES][IOOP_NUM_TYPES]; } PgStat_BktypeIO; +typedef struct PgStat_PendingIO +{ + PgStat_Counter counts[IOOBJECT_NUM_TYPES][IOCONTEXT_NUM_TYPES][IOOP_NUM_TYPES]; + instr_time pending_times[IOOBJECT_NUM_TYPES][IOCONTEXT_NUM_TYPES][IOOP_NUM_TYPES]; +} PgStat_PendingIO; + typedef struct PgStat_IO { TimestampTz stat_reset_timestamp; PgStat_BktypeIO stats[BACKEND_NUM_TYPES]; } PgStat_IO; +typedef struct PgStat_Backend +{ + TimestampTz stat_reset_timestamp; + BackendType bktype; + PgStat_BktypeIO stats; +} PgStat_Backend; typedef struct PgStat_StatDBEntry { @@ -562,6 +575,7 @@ extern void pgstat_count_io_op_time(IOObject io_object, IOContext io_context, IOOp io_op, instr_time start_time, uint32 cnt); extern PgStat_IO *pgstat_fetch_stat_io(void); +extern PgStat_Backend *pgstat_fetch_my_stat_io(void); extern const char *pgstat_get_io_context_name(IOContext io_context); extern const char *pgstat_get_io_object_name(IOObject io_object); @@ -570,6 +584,7 @@ extern bool pgstat_tracks_io_object(BackendType bktype, IOObject io_object, IOContext io_context); extern bool pgstat_tracks_io_op(BackendType bktype, IOObject io_object, IOContext io_context, IOOp io_op); +extern void pgstat_create_backend_stat(ProcNumber procnum); /* diff --git a/src/include/utils/pgstat_internal.h b/src/include/utils/pgstat_internal.h index 61b2e1f96b..44a00d4779 100644 --- a/src/include/utils/pgstat_internal.h +++ b/src/include/utils/pgstat_internal.h @@ -194,6 +194,11 @@ typedef struct PgStat_KindInfo */ bool accessed_across_databases:1; + /* + * Do serialize or not this kind of stats. + */ + bool to_serialize:1; + /* * The size of an entry in the shared stats hash table (pointed to by * PgStatShared_HashEntry->body). For fixed-numbered statistics, this is @@ -428,6 +433,11 @@ typedef struct PgStatShared_ReplSlot PgStat_StatReplSlotEntry stats; } PgStatShared_ReplSlot; +typedef struct PgStatShared_Backend +{ + PgStatShared_Common header; + PgStat_Backend stats; +} PgStatShared_Backend; /* * Central shared memory entry for the cumulative stats system. @@ -630,9 +640,12 @@ extern void pgstat_flush_io(bool nowait); extern bool pgstat_io_have_pending_cb(void); extern bool pgstat_io_flush_cb(bool nowait); +extern bool pgstat_per_backend_flush_cb(PgStat_EntryRef *entry_ref, bool nowait); extern void pgstat_io_init_shmem_cb(void *stats); extern void pgstat_io_reset_all_cb(TimestampTz ts); extern void pgstat_io_snapshot_cb(void); +extern PgStat_PendingIO *pgstat_prep_per_backend_pending(ProcNumber procnum); +extern void pgstat_backend_reset_timestamp_cb(PgStatShared_Common *header, TimestampTz ts); /* diff --git a/src/test/modules/injection_points/injection_stats.c b/src/test/modules/injection_points/injection_stats.c index d89d055913..ca4b6b6e9f 100644 --- a/src/test/modules/injection_points/injection_stats.c +++ b/src/test/modules/injection_points/injection_stats.c @@ -39,6 +39,7 @@ static bool injection_stats_flush_cb(PgStat_EntryRef *entry_ref, bool nowait); static const PgStat_KindInfo injection_stats = { .name = "injection_points", .fixed_amount = false, /* Bounded by the number of points */ + .to_serialize = true, /* Injection points are system-wide */ .accessed_across_databases = true, diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out index 2b47013f11..f92516b047 100644 --- a/src/test/regress/expected/rules.out +++ b/src/test/regress/expected/rules.out @@ -1398,6 +1398,25 @@ pg_matviews| SELECT n.nspname AS schemaname, LEFT JOIN pg_namespace n ON ((n.oid = c.relnamespace))) LEFT JOIN pg_tablespace t ON ((t.oid = c.reltablespace))) WHERE (c.relkind = 'm'::"char"); +pg_my_stat_io| SELECT backend_type, + object, + context, + reads, + read_time, + writes, + write_time, + writebacks, + writeback_time, + extends, + extend_time, + op_bytes, + hits, + evictions, + reuses, + fsyncs, + fsync_time, + stats_reset + FROM pg_stat_get_my_io() b(backend_type, object, context, reads, read_time, writes, write_time, writebacks, writeback_time, extends, extend_time, op_bytes, hits, evictions, reuses, fsyncs, fsync_time, stats_reset); pg_policies| SELECT n.nspname AS schemaname, c.relname AS tablename, pol.polname AS policyname, diff --git a/src/test/regress/expected/stats.out b/src/test/regress/expected/stats.out index 56771f83ed..846e693483 100644 --- a/src/test/regress/expected/stats.out +++ b/src/test/regress/expected/stats.out @@ -1249,7 +1249,7 @@ SELECT pg_stat_get_subscription_stats(NULL); (1 row) --- Test that the following operations are tracked in pg_stat_io: +-- Test that the following operations are tracked in pg_[my_]stat_io: -- - reads of target blocks into shared buffers -- - writes of shared buffers to permanent storage -- - extends of relations using shared buffers @@ -1261,9 +1261,14 @@ SELECT pg_stat_get_subscription_stats(NULL); -- extends. SELECT sum(extends) AS io_sum_shared_before_extends FROM pg_stat_io WHERE context = 'normal' AND object = 'relation' \gset +SELECT sum(extends) AS my_io_sum_shared_before_extends + FROM pg_my_stat_io WHERE context = 'normal' AND object = 'relation' \gset SELECT sum(writes) AS writes, sum(fsyncs) AS fsyncs FROM pg_stat_io WHERE object = 'relation' \gset io_sum_shared_before_ +SELECT sum(writes) AS writes, sum(fsyncs) AS fsyncs + FROM pg_my_stat_io + WHERE object = 'relation' \gset my_io_sum_shared_before_ CREATE TABLE test_io_shared(a int); INSERT INTO test_io_shared SELECT i FROM generate_series(1,100)i; SELECT pg_stat_force_next_flush(); @@ -1280,8 +1285,16 @@ SELECT :io_sum_shared_after_extends > :io_sum_shared_before_extends; t (1 row) +SELECT sum(extends) AS my_io_sum_shared_after_extends + FROM pg_my_stat_io WHERE context = 'normal' AND object = 'relation' \gset +SELECT :my_io_sum_shared_after_extends > :my_io_sum_shared_before_extends; + ?column? +---------- + t +(1 row) + -- After a checkpoint, there should be some additional IOCONTEXT_NORMAL writes --- and fsyncs. +-- and fsyncs in the global stats (not for the backend). -- See comment above for rationale for two explicit CHECKPOINTs. CHECKPOINT; CHECKPOINT; @@ -1301,6 +1314,23 @@ SELECT current_setting('fsync') = 'off' t (1 row) +SELECT sum(writes) AS writes, sum(fsyncs) AS fsyncs + FROM pg_my_stat_io + WHERE object = 'relation' \gset my_io_sum_shared_after_ +SELECT :my_io_sum_shared_after_writes >= :my_io_sum_shared_before_writes; + ?column? +---------- + t +(1 row) + +SELECT current_setting('fsync') = 'off' + OR (:my_io_sum_shared_after_fsyncs = :my_io_sum_shared_before_fsyncs + AND :my_io_sum_shared_after_fsyncs= 0); + ?column? +---------- + t +(1 row) + -- Change the tablespace so that the table is rewritten directly, then SELECT -- from it to cause it to be read back into shared buffers. SELECT sum(reads) AS io_sum_shared_before_reads @@ -1521,6 +1551,8 @@ SELECT pg_stat_have_stats('io', 0, 0); SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS io_stats_pre_reset FROM pg_stat_io \gset +SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS my_io_stats_pre_reset + FROM pg_my_stat_io \gset SELECT pg_stat_reset_shared('io'); pg_stat_reset_shared ---------------------- @@ -1535,6 +1567,30 @@ SELECT :io_stats_post_reset < :io_stats_pre_reset; t (1 row) +SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS my_io_stats_post_reset + FROM pg_my_stat_io \gset +-- pg_stat_reset_shared() did not reset backend IO stats +SELECT :my_io_stats_pre_reset <= :my_io_stats_post_reset; + ?column? +---------- + t +(1 row) + +-- but pg_stat_reset_single_backend_io_counters() does +SELECT pg_stat_reset_single_backend_io_counters(pg_backend_pid()); + pg_stat_reset_single_backend_io_counters +------------------------------------------ + +(1 row) + +SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS my_io_stats_post_backend_reset + FROM pg_my_stat_io \gset +SELECT :my_io_stats_pre_reset > :my_io_stats_post_backend_reset; + ?column? +---------- + t +(1 row) + -- test BRIN index doesn't block HOT update CREATE TABLE brin_hot ( id integer PRIMARY KEY, diff --git a/src/test/regress/sql/stats.sql b/src/test/regress/sql/stats.sql index 7147cc2f89..9cb14b7182 100644 --- a/src/test/regress/sql/stats.sql +++ b/src/test/regress/sql/stats.sql @@ -595,7 +595,7 @@ SELECT pg_stat_get_replication_slot(NULL); SELECT pg_stat_get_subscription_stats(NULL); --- Test that the following operations are tracked in pg_stat_io: +-- Test that the following operations are tracked in pg_[my_]stat_io: -- - reads of target blocks into shared buffers -- - writes of shared buffers to permanent storage -- - extends of relations using shared buffers @@ -609,18 +609,26 @@ SELECT pg_stat_get_subscription_stats(NULL); -- extends. SELECT sum(extends) AS io_sum_shared_before_extends FROM pg_stat_io WHERE context = 'normal' AND object = 'relation' \gset +SELECT sum(extends) AS my_io_sum_shared_before_extends + FROM pg_my_stat_io WHERE context = 'normal' AND object = 'relation' \gset SELECT sum(writes) AS writes, sum(fsyncs) AS fsyncs FROM pg_stat_io WHERE object = 'relation' \gset io_sum_shared_before_ +SELECT sum(writes) AS writes, sum(fsyncs) AS fsyncs + FROM pg_my_stat_io + WHERE object = 'relation' \gset my_io_sum_shared_before_ CREATE TABLE test_io_shared(a int); INSERT INTO test_io_shared SELECT i FROM generate_series(1,100)i; SELECT pg_stat_force_next_flush(); SELECT sum(extends) AS io_sum_shared_after_extends FROM pg_stat_io WHERE context = 'normal' AND object = 'relation' \gset SELECT :io_sum_shared_after_extends > :io_sum_shared_before_extends; +SELECT sum(extends) AS my_io_sum_shared_after_extends + FROM pg_my_stat_io WHERE context = 'normal' AND object = 'relation' \gset +SELECT :my_io_sum_shared_after_extends > :my_io_sum_shared_before_extends; -- After a checkpoint, there should be some additional IOCONTEXT_NORMAL writes --- and fsyncs. +-- and fsyncs in the global stats (not for the backend). -- See comment above for rationale for two explicit CHECKPOINTs. CHECKPOINT; CHECKPOINT; @@ -631,6 +639,14 @@ SELECT :io_sum_shared_after_writes > :io_sum_shared_before_writes; SELECT current_setting('fsync') = 'off' OR :io_sum_shared_after_fsyncs > :io_sum_shared_before_fsyncs; +SELECT sum(writes) AS writes, sum(fsyncs) AS fsyncs + FROM pg_my_stat_io + WHERE object = 'relation' \gset my_io_sum_shared_after_ +SELECT :my_io_sum_shared_after_writes >= :my_io_sum_shared_before_writes; +SELECT current_setting('fsync') = 'off' + OR (:my_io_sum_shared_after_fsyncs = :my_io_sum_shared_before_fsyncs + AND :my_io_sum_shared_after_fsyncs= 0); + -- Change the tablespace so that the table is rewritten directly, then SELECT -- from it to cause it to be read back into shared buffers. SELECT sum(reads) AS io_sum_shared_before_reads @@ -762,10 +778,21 @@ SELECT :io_sum_bulkwrite_strategy_extends_after > :io_sum_bulkwrite_strategy_ext SELECT pg_stat_have_stats('io', 0, 0); SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS io_stats_pre_reset FROM pg_stat_io \gset +SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS my_io_stats_pre_reset + FROM pg_my_stat_io \gset SELECT pg_stat_reset_shared('io'); SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS io_stats_post_reset FROM pg_stat_io \gset SELECT :io_stats_post_reset < :io_stats_pre_reset; +SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS my_io_stats_post_reset + FROM pg_my_stat_io \gset +-- pg_stat_reset_shared() did not reset backend IO stats +SELECT :my_io_stats_pre_reset <= :my_io_stats_post_reset; +-- but pg_stat_reset_single_backend_io_counters() does +SELECT pg_stat_reset_single_backend_io_counters(pg_backend_pid()); +SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS my_io_stats_post_backend_reset + FROM pg_my_stat_io \gset +SELECT :my_io_stats_pre_reset > :my_io_stats_post_backend_reset; -- test BRIN index doesn't block HOT update diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 1847bbfa95..4188d056ca 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2117,6 +2117,7 @@ PgFdwSamplingMethod PgFdwScanState PgIfAddrCallback PgStatShared_Archiver +PgStatShared_Backend PgStatShared_BgWriter PgStatShared_Checkpointer PgStatShared_Common @@ -2132,6 +2133,7 @@ PgStatShared_SLRU PgStatShared_Subscription PgStatShared_Wal PgStat_ArchiverStats +PgStat_Backend PgStat_BackendSubEntry PgStat_BgWriterStats PgStat_BktypeIO -- 2.34.1 --xszoq7AUxWDP9Wxf Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v5-0002-Merge-both-IO-stats-flush-callbacks.patch" ^ permalink raw reply [nested|flat] 25+ messages in thread
* [PATCH v6] per backend I/O statistics @ 2024-10-28 12:50 Bertrand Drouvot <[email protected]> 0 siblings, 0 replies; 25+ messages in thread From: Bertrand Drouvot @ 2024-10-28 12:50 UTC (permalink / raw) While pg_stat_io provides cluster-wide I/O statistics, this commit adds the ability to track and display per backend I/O statistics. It adds a new statistics kind, a new pg_my_stat_io view to display "my" backend I/O statistics, and 2 new functions: - pg_stat_reset_single_backend_io_counters() to be able to reset the I/O stats for a given backend pid. - pg_stat_get_backend_io() to retrieve I/O statistics for a given backend pid. The new KIND is named PGSTAT_KIND_PER_BACKEND as it could be used in the future to store other statistics (than the I/O ones) per backend. The new KIND is a variable-numbered one and has an automatic cap on the maximum number of entries (as its hash key contains the proc number). There is no need to write the per backend I/O stats to disk (no point to see stats for backends that do not exist anymore after a re-start), so using "write_to_file = false". Note that per backend I/O statistics are not collected for the checkpointer, the background writer, the startup process and the autovacuum launcher as those are already visible in pg_stat_io and there is only one of those. XXX: Bump catalog version needs to be done. --- doc/src/sgml/config.sgml | 4 +- doc/src/sgml/monitoring.sgml | 63 ++++++ src/backend/catalog/system_functions.sql | 2 + src/backend/catalog/system_views.sql | 22 +++ src/backend/utils/activity/Makefile | 1 + src/backend/utils/activity/backend_status.c | 4 + src/backend/utils/activity/meson.build | 1 + src/backend/utils/activity/pgstat.c | 19 +- src/backend/utils/activity/pgstat_backend.c | 176 +++++++++++++++++ src/backend/utils/activity/pgstat_io.c | 37 +++- src/backend/utils/adt/pgstatfuncs.c | 208 ++++++++++++++++++++ src/include/catalog/pg_proc.dat | 14 ++ src/include/pgstat.h | 32 ++- src/include/utils/pgstat_internal.h | 12 ++ src/test/regress/expected/rules.out | 19 ++ src/test/regress/expected/stats.out | 85 +++++++- src/test/regress/sql/stats.sql | 47 ++++- src/tools/pgindent/typedefs.list | 3 + 18 files changed, 729 insertions(+), 20 deletions(-) 11.6% doc/src/sgml/ 27.0% src/backend/utils/activity/ 22.4% src/backend/utils/adt/ 4.0% src/include/catalog/ 5.9% src/include/ 14.9% src/test/regress/expected/ 11.4% src/test/regress/sql/ diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index a84e60c09b..20a0862661 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -8365,7 +8365,9 @@ COPY postgres_log FROM '/full/path/to/logfile.csv' WITH csv; displayed in <link linkend="monitoring-pg-stat-database-view"> <structname>pg_stat_database</structname></link>, <link linkend="monitoring-pg-stat-io-view"> - <structname>pg_stat_io</structname></link>, in the output of + <structname>pg_stat_io</structname></link>, + <link linkend="monitoring-pg-my-stat-io-view"> + <structname>pg_my_stat_io</structname></link>, in the output of <xref linkend="sql-explain"/> when the <literal>BUFFERS</literal> option is used, in the output of <xref linkend="sql-vacuum"/> when the <literal>VERBOSE</literal> option is used, by autovacuum diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml index 840d7f8161..8e7a7a0f36 100644 --- a/doc/src/sgml/monitoring.sgml +++ b/doc/src/sgml/monitoring.sgml @@ -488,6 +488,16 @@ postgres 27093 0.0 0.0 30096 2752 ? Ss 11:34 0:00 postgres: ser </entry> </row> + <row> + <entry><structname>pg_my_stat_io</structname><indexterm><primary>pg_my_stat_io</primary></indexterm></entry> + <entry> + One row for each combination of context and target object containing + my backend I/O statistics. + See <link linkend="monitoring-pg-my-stat-io-view"> + <structname>pg_my_stat_io</structname></link> for details. + </entry> + </row> + <row> <entry><structname>pg_stat_replication_slots</structname><indexterm><primary>pg_stat_replication_slots</primary></indexterm></entry> <entry>One row per replication slot, showing statistics about the @@ -2946,7 +2956,23 @@ description | Waiting for a newly initialized WAL file to reach durable storage </para> </note> + </sect2> + + <sect2 id="monitoring-pg-my-stat-io-view"> + <title><structname>pg_my_stat_io</structname></title> + + <indexterm> + <primary>pg_my_stat_io</primary> + </indexterm> + <para> + The <structname>pg_my_stat_io</structname> view will contain one row for each + combination of target I/O object and I/O context, showing + my backend I/O statistics. Combinations which do not make sense are + omitted. The fields are exactly the same as the ones in the <link + linkend="monitoring-pg-stat-io-view"> <structname>pg_stat_io</structname></link> + view. + </para> </sect2> @@ -4790,6 +4816,25 @@ description | Waiting for a newly initialized WAL file to reach durable storage </para></entry> </row> + <row> + <entry role="func_table_entry"><para role="func_signature"> + <indexterm> + <primary>pg_stat_get_backend_io</primary> + </indexterm> + <function>pg_stat_get_backend_io</function> ( <type>integer</type> ) + <returnvalue>setof record</returnvalue> + </para> + <para> + Returns I/O statistics about the backend with the specified + process ID. The output fields are exactly the same as the ones in the + <link linkend="monitoring-pg-stat-io-view"> <structname>pg_stat_io</structname></link> + view. The function does not return I/O statistics for the checkpointer, + the background writer, the startup process and the autovacuum launcher + as they are already visible in the <link linkend="monitoring-pg-stat-io-view"> <structname>pg_stat_io</structname></link> + view and there is only one of those. + </para></entry> + </row> + <row> <entry role="func_table_entry"><para role="func_signature"> <indexterm> @@ -4971,6 +5016,24 @@ description | Waiting for a newly initialized WAL file to reach durable storage </para></entry> </row> + <row> + <entry role="func_table_entry"><para role="func_signature"> + <indexterm> + <primary>pg_stat_reset_single_backend_io_counters</primary> + </indexterm> + <function>pg_stat_reset_single_backend_io_counters</function> ( <type>integer</type> ) + <returnvalue>void</returnvalue> + </para> + <para> + Resets I/O statistics for a single backend with the specified process ID + to zero. + </para> + <para> + This function is restricted to superusers by default, but other users + can be granted EXECUTE to run the function. + </para></entry> + </row> + <row> <entry role="func_table_entry"><para role="func_signature"> <indexterm> diff --git a/src/backend/catalog/system_functions.sql b/src/backend/catalog/system_functions.sql index c51dfca802..1b21b242cd 100644 --- a/src/backend/catalog/system_functions.sql +++ b/src/backend/catalog/system_functions.sql @@ -711,6 +711,8 @@ REVOKE EXECUTE ON FUNCTION pg_stat_reset_single_table_counters(oid) FROM public; REVOKE EXECUTE ON FUNCTION pg_stat_reset_single_function_counters(oid) FROM public; +REVOKE EXECUTE ON FUNCTION pg_stat_reset_single_backend_io_counters(integer) FROM public; + REVOKE EXECUTE ON FUNCTION pg_stat_reset_replication_slot(text) FROM public; REVOKE EXECUTE ON FUNCTION pg_stat_have_stats(text, oid, int8) FROM public; diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index da9a8fe99f..fce3b791bb 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1172,6 +1172,28 @@ SELECT b.stats_reset FROM pg_stat_get_io() b; +CREATE VIEW pg_my_stat_io AS +SELECT + b.backend_type, + b.object, + b.context, + b.reads, + b.read_time, + b.writes, + b.write_time, + b.writebacks, + b.writeback_time, + b.extends, + b.extend_time, + b.op_bytes, + b.hits, + b.evictions, + b.reuses, + b.fsyncs, + b.fsync_time, + b.stats_reset +FROM pg_stat_get_backend_io(NULL) b; + CREATE VIEW pg_stat_wal AS SELECT w.wal_records, diff --git a/src/backend/utils/activity/Makefile b/src/backend/utils/activity/Makefile index b9fd66ea17..24b64a2742 100644 --- a/src/backend/utils/activity/Makefile +++ b/src/backend/utils/activity/Makefile @@ -20,6 +20,7 @@ OBJS = \ backend_status.o \ pgstat.o \ pgstat_archiver.o \ + pgstat_backend.o \ pgstat_bgwriter.o \ pgstat_checkpointer.o \ pgstat_database.o \ diff --git a/src/backend/utils/activity/backend_status.c b/src/backend/utils/activity/backend_status.c index bdb3a296ca..249439b990 100644 --- a/src/backend/utils/activity/backend_status.c +++ b/src/backend/utils/activity/backend_status.c @@ -249,6 +249,10 @@ pgstat_beinit(void) Assert(MyProcNumber >= 0 && MyProcNumber < NumBackendStatSlots); MyBEEntry = &BackendStatusArray[MyProcNumber]; + /* Create the per-backend statistics entry */ + if (pgstat_tracks_per_backend_bktype(MyBackendType)) + pgstat_create_backend_stat(MyProcNumber); + /* Set up a process-exit hook to clean up */ on_shmem_exit(pgstat_beshutdown_hook, 0); } diff --git a/src/backend/utils/activity/meson.build b/src/backend/utils/activity/meson.build index f73c22905c..380d3dd70c 100644 --- a/src/backend/utils/activity/meson.build +++ b/src/backend/utils/activity/meson.build @@ -5,6 +5,7 @@ backend_sources += files( 'backend_status.c', 'pgstat.c', 'pgstat_archiver.c', + 'pgstat_backend.c', 'pgstat_bgwriter.c', 'pgstat_checkpointer.c', 'pgstat_database.c', diff --git a/src/backend/utils/activity/pgstat.c b/src/backend/utils/activity/pgstat.c index 6f8d237826..1449d6a3f4 100644 --- a/src/backend/utils/activity/pgstat.c +++ b/src/backend/utils/activity/pgstat.c @@ -77,6 +77,7 @@ * * Each statistics kind is handled in a dedicated file: * - pgstat_archiver.c + * - pgstat_backend.c * - pgstat_bgwriter.c * - pgstat_checkpointer.c * - pgstat_database.c @@ -358,6 +359,22 @@ static const PgStat_KindInfo pgstat_kind_builtin_infos[PGSTAT_KIND_BUILTIN_SIZE] .reset_timestamp_cb = pgstat_subscription_reset_timestamp_cb, }, + [PGSTAT_KIND_PER_BACKEND] = { + .name = "per-backend", + + .fixed_amount = false, + .write_to_file = false, + + .accessed_across_databases = true, + + .shared_size = sizeof(PgStatShared_Backend), + .shared_data_off = offsetof(PgStatShared_Backend, stats), + .shared_data_len = sizeof(((PgStatShared_Backend *) 0)->stats), + .pending_size = sizeof(PgStat_BackendPendingIO), + + .flush_pending_cb = pgstat_per_backend_flush_cb, + .reset_timestamp_cb = pgstat_backend_reset_timestamp_cb, + }, /* stats for fixed-numbered (mostly 1) objects */ @@ -768,7 +785,7 @@ pgstat_report_stat(bool force) partial_flush = false; - /* flush database / relation / function / ... stats */ + /* flush database / relation / function / backend / ... stats */ partial_flush |= pgstat_flush_pending_entries(nowait); /* flush of fixed-numbered stats */ diff --git a/src/backend/utils/activity/pgstat_backend.c b/src/backend/utils/activity/pgstat_backend.c new file mode 100644 index 0000000000..f079f15ff2 --- /dev/null +++ b/src/backend/utils/activity/pgstat_backend.c @@ -0,0 +1,176 @@ +/* ------------------------------------------------------------------------- + * + * pgstat_backend.c + * Implementation of per-backend statistics. + * + * This file contains the implementation of per-backend statistics. It is kept + * separate from pgstat.c to enforce the line between the statistics access / + * storage implementation and the details about individual types of statistics. + * + * Copyright (c) 2024, PostgreSQL Global Development Group + * + * IDENTIFICATION + * src/backend/utils/activity/pgstat_backend.c + * ------------------------------------------------------------------------- + */ + +#include "postgres.h" + +#include "utils/pgstat_internal.h" + +/* + * Returns my backend IO stats. + */ +PgStat_Backend * +pgstat_fetch_my_stat_io(void) +{ + return (PgStat_Backend *) + pgstat_fetch_entry(PGSTAT_KIND_PER_BACKEND, InvalidOid, MyProcNumber); +} + +/* + * Returns other backend's IO stats. + */ +PgStat_Backend * +pgstat_fetch_proc_stat_io(ProcNumber procNumber) +{ + PgStat_Backend *backend_entry; + + backend_entry = (PgStat_Backend *) pgstat_fetch_entry(PGSTAT_KIND_PER_BACKEND, + InvalidOid, procNumber); + + return backend_entry; +} + +/* + * Flush out locally pending backend statistics + * + * If no stats have been recorded, this function returns false. + */ +bool +pgstat_per_backend_flush_cb(PgStat_EntryRef *entry_ref, bool nowait) +{ + PgStatShared_Backend *shbackendioent; + PgStat_BackendPendingIO *pendingent; + PgStat_BktypeIO *bktype_shstats; + + if (!pgstat_lock_entry(entry_ref, nowait)) + return false; + + shbackendioent = (PgStatShared_Backend *) entry_ref->shared_stats; + bktype_shstats = &shbackendioent->stats.stats; + pendingent = (PgStat_BackendPendingIO *) entry_ref->pending; + + for (int io_object = 0; io_object < IOOBJECT_NUM_TYPES; io_object++) + { + for (int io_context = 0; io_context < IOCONTEXT_NUM_TYPES; io_context++) + { + for (int io_op = 0; io_op < IOOP_NUM_TYPES; io_op++) + { + instr_time time; + + bktype_shstats->counts[io_object][io_context][io_op] += + pendingent->counts[io_object][io_context][io_op]; + + time = pendingent->pending_times[io_object][io_context][io_op]; + + bktype_shstats->times[io_object][io_context][io_op] += + INSTR_TIME_GET_MICROSEC(time); + } + } + } + + pgstat_unlock_entry(entry_ref); + + return true; +} + +/* + * Create the per-backend statistics entry for procnum. + */ +void +pgstat_create_backend_stat(ProcNumber procnum) +{ + PgStat_EntryRef *entry_ref; + PgStatShared_Backend *shstatent; + + entry_ref = pgstat_prep_pending_entry(PGSTAT_KIND_PER_BACKEND, InvalidOid, + procnum, NULL); + + shstatent = (PgStatShared_Backend *) entry_ref->shared_stats; + + /* + * NB: need to accept that there might be stats from an older backend, + * e.g. if we previously used this proc number. + */ + memset(&shstatent->stats, 0, sizeof(shstatent->stats)); +} + +/* + * Find or create a local PgStat_BackendPendingIO entry for procnum. + */ +PgStat_BackendPendingIO * +pgstat_prep_per_backend_pending(ProcNumber procnum) +{ + PgStat_EntryRef *entry_ref; + + entry_ref = pgstat_prep_pending_entry(PGSTAT_KIND_PER_BACKEND, InvalidOid, + procnum, NULL); + + return entry_ref->pending; +} + +/* + * per-backend statistics are not collected for all BackendTypes. + * + * The following BackendTypes do not participate in the per-backend stats + * subsystem: + * - The same and for the same reasons as in pgstat_tracks_io_bktype(). + * - B_BG_WRITER, B_CHECKPOINTER, B_STARTUP and B_AUTOVAC_LAUNCHER because their + * I/O stats are already visible in pg_stat_io and there is only one of those. + * + * Function returns true if BackendType participates in the per-backend stats + * subsystem for IO and false if it does not. + * + * When adding a new BackendType, also consider adding relevant restrictions to + * pgstat_tracks_io_object() and pgstat_tracks_io_op(). + */ +bool +pgstat_tracks_per_backend_bktype(BackendType bktype) +{ + /* + * List every type so that new backend types trigger a warning about + * needing to adjust this switch. + */ + switch (bktype) + { + case B_INVALID: + case B_AUTOVAC_LAUNCHER: + case B_DEAD_END_BACKEND: + case B_ARCHIVER: + case B_LOGGER: + case B_WAL_RECEIVER: + case B_WAL_WRITER: + case B_WAL_SUMMARIZER: + case B_BG_WRITER: + case B_CHECKPOINTER: + case B_STARTUP: + return false; + + case B_AUTOVAC_WORKER: + case B_BACKEND: + case B_BG_WORKER: + case B_STANDALONE_BACKEND: + case B_SLOTSYNC_WORKER: + case B_WAL_SENDER: + return true; + } + + return false; +} + +void +pgstat_backend_reset_timestamp_cb(PgStatShared_Common *header, TimestampTz ts) +{ + ((PgStatShared_Backend *) header)->stats.stat_reset_timestamp = ts; +} diff --git a/src/backend/utils/activity/pgstat_io.c b/src/backend/utils/activity/pgstat_io.c index f9883af2b3..1af5b75df6 100644 --- a/src/backend/utils/activity/pgstat_io.c +++ b/src/backend/utils/activity/pgstat_io.c @@ -20,13 +20,7 @@ #include "storage/bufmgr.h" #include "utils/pgstat_internal.h" - -typedef struct PgStat_PendingIO -{ - PgStat_Counter counts[IOOBJECT_NUM_TYPES][IOCONTEXT_NUM_TYPES][IOOP_NUM_TYPES]; - instr_time pending_times[IOOBJECT_NUM_TYPES][IOCONTEXT_NUM_TYPES][IOOP_NUM_TYPES]; -} PgStat_PendingIO; - +typedef PgStat_BackendPendingIO PgStat_PendingIO; static PgStat_PendingIO PendingIOStats; static bool have_iostats = false; @@ -87,6 +81,14 @@ pgstat_count_io_op_n(IOObject io_object, IOContext io_context, IOOp io_op, uint3 Assert((unsigned int) io_op < IOOP_NUM_TYPES); Assert(pgstat_tracks_io_op(MyBackendType, io_object, io_context, io_op)); + if (pgstat_tracks_per_backend_bktype(MyBackendType)) + { + PgStat_PendingIO *entry_ref; + + entry_ref = pgstat_prep_per_backend_pending(MyProcNumber); + entry_ref->counts[io_object][io_context][io_op] += cnt; + } + PendingIOStats.counts[io_object][io_context][io_op] += cnt; have_iostats = true; @@ -122,6 +124,7 @@ void pgstat_count_io_op_time(IOObject io_object, IOContext io_context, IOOp io_op, instr_time start_time, uint32 cnt) { + if (track_io_timing) { instr_time io_time; @@ -148,6 +151,15 @@ pgstat_count_io_op_time(IOObject io_object, IOContext io_context, IOOp io_op, INSTR_TIME_ADD(PendingIOStats.pending_times[io_object][io_context][io_op], io_time); + + if (pgstat_tracks_per_backend_bktype(MyBackendType)) + { + PgStat_PendingIO *entry_ref; + + entry_ref = pgstat_prep_per_backend_pending(MyProcNumber); + INSTR_TIME_ADD(entry_ref->pending_times[io_object][io_context][io_op], + io_time); + } } pgstat_count_io_op_n(io_object, io_context, io_op, cnt); @@ -171,12 +183,21 @@ pgstat_io_have_pending_cb(void) } /* - * Simpler wrapper of pgstat_io_flush_cb() + * Simpler wrapper of pgstat_io_flush_cb() and pgstat_per_backend_flush_cb(). */ void pgstat_flush_io(bool nowait) { + PgStat_EntryRef *entry_ref; + (void) pgstat_io_flush_cb(nowait); + + if (pgstat_tracks_per_backend_bktype(MyBackendType)) + { + entry_ref = pgstat_get_entry_ref(PGSTAT_KIND_PER_BACKEND, InvalidOid, + MyProcNumber, false, NULL); + (void) pgstat_per_backend_flush_cb(entry_ref, nowait); + } } /* diff --git a/src/backend/utils/adt/pgstatfuncs.c b/src/backend/utils/adt/pgstatfuncs.c index 60a397dc56..0302970246 100644 --- a/src/backend/utils/adt/pgstatfuncs.c +++ b/src/backend/utils/adt/pgstatfuncs.c @@ -1480,6 +1480,165 @@ pg_stat_get_io(PG_FUNCTION_ARGS) return (Datum) 0; } +Datum +pg_stat_get_backend_io(PG_FUNCTION_ARGS) +{ + ReturnSetInfo *rsinfo; + PgStat_Backend *backend_stats; + Datum bktype_desc; + PgStat_BktypeIO *bktype_stats; + BackendType bktype; + Datum reset_time; + int num_backends = pgstat_fetch_stat_numbackends(); + int curr_backend; + int pid; + + /* just to keep compiler quiet */ + bktype = B_INVALID; + + InitMaterializedSRF(fcinfo, 0); + rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; + + if (PG_ARGISNULL(0) || PG_GETARG_INT32(0) == MyProcPid) + { + backend_stats = pgstat_fetch_my_stat_io(); + pid = MyProcPid; + bktype = MyBackendType; + } + else + { + PGPROC *proc; + ProcNumber procNumber; + + pid = PG_GETARG_INT32(0); + proc = BackendPidGetProc(pid); + + /* + * Maybe an auxiliary process? That should not be possible, due to + * pgstat_tracks_per_backend_bktype() though. + */ + if (proc == NULL) + proc = AuxiliaryPidGetProc(pid); + + if (proc != NULL) + { + procNumber = GetNumberFromPGProc(proc); + backend_stats = pgstat_fetch_proc_stat_io(procNumber); + + if (!backend_stats) + return (Datum) 0; + } + else + return (Datum) 0; + } + + /* Look for the backend type */ + for (curr_backend = 1; curr_backend <= num_backends; curr_backend++) + { + LocalPgBackendStatus *local_beentry; + PgBackendStatus *beentry; + + /* Get the next one in the list */ + local_beentry = pgstat_get_local_beentry_by_index(curr_backend); + beentry = &local_beentry->backendStatus; + + /* looking for specific PID, ignore all the others */ + if (beentry->st_procpid != pid) + continue; + + bktype = beentry->st_backendType; + break; + } + + bktype_desc = CStringGetTextDatum(GetBackendTypeDesc(bktype)); + bktype_stats = &backend_stats->stats; + reset_time = TimestampTzGetDatum(backend_stats->stat_reset_timestamp); + + /* + * In Assert builds, we can afford an extra loop through all of the + * counters checking that only expected stats are non-zero, since it keeps + * the non-Assert code cleaner. + */ + Assert(pgstat_bktype_io_stats_valid(bktype_stats, bktype)); + + for (int io_obj = 0; io_obj < IOOBJECT_NUM_TYPES; io_obj++) + { + const char *obj_name = pgstat_get_io_object_name(io_obj); + + for (int io_context = 0; io_context < IOCONTEXT_NUM_TYPES; io_context++) + { + const char *context_name = pgstat_get_io_context_name(io_context); + + Datum values[IO_NUM_COLUMNS] = {0}; + bool nulls[IO_NUM_COLUMNS] = {0}; + + /* + * Some combinations of BackendType, IOObject, and IOContext are + * not valid for any type of IOOp. In such cases, omit the entire + * row from the view. + */ + if (!pgstat_tracks_io_object(bktype, io_obj, io_context)) + continue; + + values[IO_COL_BACKEND_TYPE] = bktype_desc; + values[IO_COL_CONTEXT] = CStringGetTextDatum(context_name); + values[IO_COL_OBJECT] = CStringGetTextDatum(obj_name); + if (backend_stats->stat_reset_timestamp != 0) + values[IO_COL_RESET_TIME] = reset_time; + else + nulls[IO_COL_RESET_TIME] = true; + + /* + * Hard-code this to the value of BLCKSZ for now. Future values + * could include XLOG_BLCKSZ, once WAL IO is tracked, and constant + * multipliers, once non-block-oriented IO (e.g. temporary file + * IO) is tracked. + */ + values[IO_COL_CONVERSION] = Int64GetDatum(BLCKSZ); + + for (int io_op = 0; io_op < IOOP_NUM_TYPES; io_op++) + { + int op_idx = pgstat_get_io_op_index(io_op); + int time_idx = pgstat_get_io_time_index(io_op); + + /* + * Some combinations of BackendType and IOOp, of IOContext and + * IOOp, and of IOObject and IOOp are not tracked. Set these + * cells in the view NULL. + */ + if (pgstat_tracks_io_op(bktype, io_obj, io_context, io_op)) + { + PgStat_Counter count = + bktype_stats->counts[io_obj][io_context][io_op]; + + values[op_idx] = Int64GetDatum(count); + } + else + nulls[op_idx] = true; + + /* not every operation is timed */ + if (time_idx == IO_COL_INVALID) + continue; + + if (!nulls[op_idx]) + { + PgStat_Counter time = + bktype_stats->times[io_obj][io_context][io_op]; + + values[time_idx] = Float8GetDatum(pg_stat_us_to_ms(time)); + } + else + nulls[time_idx] = true; + } + + tuplestore_putvalues(rsinfo->setResult, rsinfo->setDesc, + values, nulls); + } + } + + return (Datum) 0; +} + /* * Returns statistics of WAL activity */ @@ -1785,6 +1944,55 @@ pg_stat_reset_single_function_counters(PG_FUNCTION_ARGS) PG_RETURN_VOID(); } +Datum +pg_stat_reset_single_backend_io_counters(PG_FUNCTION_ARGS) +{ + PGPROC *proc; + BackendType bktype; + int backend_pid = PG_GETARG_INT32(0); + int num_backends = pgstat_fetch_stat_numbackends(); + int curr_backend; + + proc = BackendPidGetProc(backend_pid); + + /* + * Maybe an auxiliary process? That should not be possible, due to + * pgstat_tracks_per_backend_bktype() though. + */ + if (proc == NULL) + proc = AuxiliaryPidGetProc(backend_pid); + + /* Check that IO stats are collected for this backend type */ + if (proc) + { + for (curr_backend = 1; curr_backend <= num_backends; curr_backend++) + { + LocalPgBackendStatus *local_beentry; + PgBackendStatus *beentry; + + /* Get the next one in the list */ + local_beentry = pgstat_get_local_beentry_by_index(curr_backend); + beentry = &local_beentry->backendStatus; + + /* looking for specific PID, ignore all the others */ + if (beentry->st_procpid != backend_pid) + continue; + + bktype = beentry->st_backendType; + + if (!pgstat_tracks_per_backend_bktype(bktype)) + PG_RETURN_VOID(); + else + { + pgstat_reset(PGSTAT_KIND_PER_BACKEND, InvalidOid, GetNumberFromPGProc(proc)); + break; + } + } + } + + PG_RETURN_VOID(); +} + /* Reset SLRU counters (a specific one or all of them). */ Datum pg_stat_reset_slru(PG_FUNCTION_ARGS) diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index cbbe8acd38..6e5c2082a7 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -5913,6 +5913,15 @@ proargnames => '{backend_type,object,context,reads,read_time,writes,write_time,writebacks,writeback_time,extends,extend_time,op_bytes,hits,evictions,reuses,fsyncs,fsync_time,stats_reset}', prosrc => 'pg_stat_get_io' }, +{ oid => '8806', descr => 'statistics: per backend IO statistics', + proname => 'pg_stat_get_backend_io', prorows => '5', proretset => 't', + provolatile => 'v', proparallel => 'r', prorettype => 'record', + proargtypes => 'int4', proisstrict => 'f', + proallargtypes => '{int4,text,text,text,int8,float8,int8,float8,int8,float8,int8,float8,int8,int8,int8,int8,int8,float8,timestamptz}', + proargmodes => '{i,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o}', + proargnames => '{backend_pid,backend_type,object,context,reads,read_time,writes,write_time,writebacks,writeback_time,extends,extend_time,op_bytes,hits,evictions,reuses,fsyncs,fsync_time,stats_reset}', + prosrc => 'pg_stat_get_backend_io' }, + { oid => '1136', descr => 'statistics: information about WAL activity', proname => 'pg_stat_get_wal', proisstrict => 'f', provolatile => 's', proparallel => 'r', prorettype => 'record', proargtypes => '', @@ -6052,6 +6061,11 @@ proname => 'pg_stat_reset_single_function_counters', provolatile => 'v', prorettype => 'void', proargtypes => 'oid', prosrc => 'pg_stat_reset_single_function_counters' }, +{ oid => '9987', + descr => 'statistics: reset collected IO statistics for a single backend', + proname => 'pg_stat_reset_single_backend_io_counters', provolatile => 'v', + prorettype => 'void', proargtypes => 'int4', + prosrc => 'pg_stat_reset_single_backend_io_counters' }, { oid => '2307', descr => 'statistics: reset collected statistics for a single SLRU', proname => 'pg_stat_reset_slru', proisstrict => 'f', provolatile => 'v', diff --git a/src/include/pgstat.h b/src/include/pgstat.h index 59c28b4aca..c5151c8e6f 100644 --- a/src/include/pgstat.h +++ b/src/include/pgstat.h @@ -49,14 +49,15 @@ #define PGSTAT_KIND_FUNCTION 3 /* per-function statistics */ #define PGSTAT_KIND_REPLSLOT 4 /* per-slot statistics */ #define PGSTAT_KIND_SUBSCRIPTION 5 /* per-subscription statistics */ +#define PGSTAT_KIND_PER_BACKEND 6 /* stats for fixed-numbered objects */ -#define PGSTAT_KIND_ARCHIVER 6 -#define PGSTAT_KIND_BGWRITER 7 -#define PGSTAT_KIND_CHECKPOINTER 8 -#define PGSTAT_KIND_IO 9 -#define PGSTAT_KIND_SLRU 10 -#define PGSTAT_KIND_WAL 11 +#define PGSTAT_KIND_ARCHIVER 7 +#define PGSTAT_KIND_BGWRITER 8 +#define PGSTAT_KIND_CHECKPOINTER 9 +#define PGSTAT_KIND_IO 10 +#define PGSTAT_KIND_SLRU 11 +#define PGSTAT_KIND_WAL 12 #define PGSTAT_KIND_BUILTIN_MIN PGSTAT_KIND_DATABASE #define PGSTAT_KIND_BUILTIN_MAX PGSTAT_KIND_WAL @@ -347,12 +348,23 @@ typedef struct PgStat_BktypeIO PgStat_Counter times[IOOBJECT_NUM_TYPES][IOCONTEXT_NUM_TYPES][IOOP_NUM_TYPES]; } PgStat_BktypeIO; +typedef struct PgStat_BackendPendingIO +{ + PgStat_Counter counts[IOOBJECT_NUM_TYPES][IOCONTEXT_NUM_TYPES][IOOP_NUM_TYPES]; + instr_time pending_times[IOOBJECT_NUM_TYPES][IOCONTEXT_NUM_TYPES][IOOP_NUM_TYPES]; +} PgStat_BackendPendingIO; + typedef struct PgStat_IO { TimestampTz stat_reset_timestamp; PgStat_BktypeIO stats[BACKEND_NUM_TYPES]; } PgStat_IO; +typedef struct PgStat_Backend +{ + TimestampTz stat_reset_timestamp; + PgStat_BktypeIO stats; +} PgStat_Backend; typedef struct PgStat_StatDBEntry { @@ -534,6 +546,14 @@ extern bool pgstat_have_entry(PgStat_Kind kind, Oid dboid, uint64 objid); extern void pgstat_report_archiver(const char *xlog, bool failed); extern PgStat_ArchiverStats *pgstat_fetch_stat_archiver(void); +/* + * Functions in pgstat_backend.c + */ + +extern PgStat_Backend *pgstat_fetch_my_stat_io(void); +extern PgStat_Backend *pgstat_fetch_proc_stat_io(ProcNumber procNumber); +extern bool pgstat_tracks_per_backend_bktype(BackendType bktype); +extern void pgstat_create_backend_stat(ProcNumber procnum); /* * Functions in pgstat_bgwriter.c diff --git a/src/include/utils/pgstat_internal.h b/src/include/utils/pgstat_internal.h index 7338bc1e28..625bb55864 100644 --- a/src/include/utils/pgstat_internal.h +++ b/src/include/utils/pgstat_internal.h @@ -450,6 +450,11 @@ typedef struct PgStatShared_ReplSlot PgStat_StatReplSlotEntry stats; } PgStatShared_ReplSlot; +typedef struct PgStatShared_Backend +{ + PgStatShared_Common header; + PgStat_Backend stats; +} PgStatShared_Backend; /* * Central shared memory entry for the cumulative stats system. @@ -604,6 +609,13 @@ extern void pgstat_archiver_init_shmem_cb(void *stats); extern void pgstat_archiver_reset_all_cb(TimestampTz ts); extern void pgstat_archiver_snapshot_cb(void); +/* + * Functions in pgstat_backend.c + */ + +extern PgStat_BackendPendingIO *pgstat_prep_per_backend_pending(ProcNumber procnum); +extern bool pgstat_per_backend_flush_cb(PgStat_EntryRef *entry_ref, bool nowait); +extern void pgstat_backend_reset_timestamp_cb(PgStatShared_Common *header, TimestampTz ts); /* * Functions in pgstat_bgwriter.c diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out index 3014d047fe..3922675bbf 100644 --- a/src/test/regress/expected/rules.out +++ b/src/test/regress/expected/rules.out @@ -1398,6 +1398,25 @@ pg_matviews| SELECT n.nspname AS schemaname, LEFT JOIN pg_namespace n ON ((n.oid = c.relnamespace))) LEFT JOIN pg_tablespace t ON ((t.oid = c.reltablespace))) WHERE (c.relkind = 'm'::"char"); +pg_my_stat_io| SELECT backend_type, + object, + context, + reads, + read_time, + writes, + write_time, + writebacks, + writeback_time, + extends, + extend_time, + op_bytes, + hits, + evictions, + reuses, + fsyncs, + fsync_time, + stats_reset + FROM pg_stat_get_backend_io(NULL::integer) b(backend_type, object, context, reads, read_time, writes, write_time, writebacks, writeback_time, extends, extend_time, op_bytes, hits, evictions, reuses, fsyncs, fsync_time, stats_reset); pg_policies| SELECT n.nspname AS schemaname, c.relname AS tablename, pol.polname AS policyname, diff --git a/src/test/regress/expected/stats.out b/src/test/regress/expected/stats.out index 56771f83ed..7f2fd873b9 100644 --- a/src/test/regress/expected/stats.out +++ b/src/test/regress/expected/stats.out @@ -1249,7 +1249,7 @@ SELECT pg_stat_get_subscription_stats(NULL); (1 row) --- Test that the following operations are tracked in pg_stat_io: +-- Test that the following operations are tracked in pg_[my_]stat_io: -- - reads of target blocks into shared buffers -- - writes of shared buffers to permanent storage -- - extends of relations using shared buffers @@ -1259,11 +1259,24 @@ SELECT pg_stat_get_subscription_stats(NULL); -- be sure of the state of shared buffers at the point the test is run. -- Create a regular table and insert some data to generate IOCONTEXT_NORMAL -- extends. +SELECT pid AS checkpointer_pid FROM pg_stat_activity + WHERE backend_type = 'checkpointer' \gset SELECT sum(extends) AS io_sum_shared_before_extends FROM pg_stat_io WHERE context = 'normal' AND object = 'relation' \gset +SELECT sum(extends) AS my_io_sum_shared_before_extends + FROM pg_my_stat_io WHERE context = 'normal' AND object = 'relation' \gset +SELECT sum(extends) AS backend_io_sum_shared_before_extends + FROM pg_stat_get_backend_io(pg_backend_pid()) + WHERE context = 'normal' AND object = 'relation' \gset SELECT sum(writes) AS writes, sum(fsyncs) AS fsyncs FROM pg_stat_io WHERE object = 'relation' \gset io_sum_shared_before_ +SELECT sum(writes) AS writes, sum(fsyncs) AS fsyncs + FROM pg_my_stat_io + WHERE object = 'relation' \gset my_io_sum_shared_before_ +SELECT sum(writes) AS writes, sum(fsyncs) AS fsyncs + FROM pg_stat_get_backend_io(pg_backend_pid()) + WHERE object = 'relation' \gset backend_io_sum_shared_before_ CREATE TABLE test_io_shared(a int); INSERT INTO test_io_shared SELECT i FROM generate_series(1,100)i; SELECT pg_stat_force_next_flush(); @@ -1280,8 +1293,25 @@ SELECT :io_sum_shared_after_extends > :io_sum_shared_before_extends; t (1 row) +SELECT sum(extends) AS my_io_sum_shared_after_extends + FROM pg_my_stat_io WHERE context = 'normal' AND object = 'relation' \gset +SELECT :my_io_sum_shared_after_extends > :my_io_sum_shared_before_extends; + ?column? +---------- + t +(1 row) + +SELECT sum(extends) AS backend_io_sum_shared_after_extends + FROM pg_stat_get_backend_io(pg_backend_pid()) + WHERE context = 'normal' AND object = 'relation' \gset +SELECT :backend_io_sum_shared_after_extends > :backend_io_sum_shared_before_extends; + ?column? +---------- + t +(1 row) + -- After a checkpoint, there should be some additional IOCONTEXT_NORMAL writes --- and fsyncs. +-- and fsyncs in the global stats (not for the backend). -- See comment above for rationale for two explicit CHECKPOINTs. CHECKPOINT; CHECKPOINT; @@ -1301,6 +1331,31 @@ SELECT current_setting('fsync') = 'off' t (1 row) +SELECT sum(writes) AS writes, sum(fsyncs) AS fsyncs + FROM pg_my_stat_io + WHERE object = 'relation' \gset my_io_sum_shared_after_ +SELECT :my_io_sum_shared_after_writes >= :my_io_sum_shared_before_writes; + ?column? +---------- + t +(1 row) + +SELECT current_setting('fsync') = 'off' + OR (:my_io_sum_shared_after_fsyncs = :my_io_sum_shared_before_fsyncs + AND :my_io_sum_shared_after_fsyncs= 0); + ?column? +---------- + t +(1 row) + +-- Don't return any rows if querying other backend's stats that are excluded +-- from the per-backend stats collection (like the checkpointer). +SELECT count(1) = 0 FROM pg_stat_get_backend_io(:checkpointer_pid); + ?column? +---------- + t +(1 row) + -- Change the tablespace so that the table is rewritten directly, then SELECT -- from it to cause it to be read back into shared buffers. SELECT sum(reads) AS io_sum_shared_before_reads @@ -1521,6 +1576,8 @@ SELECT pg_stat_have_stats('io', 0, 0); SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS io_stats_pre_reset FROM pg_stat_io \gset +SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS my_io_stats_pre_reset + FROM pg_my_stat_io \gset SELECT pg_stat_reset_shared('io'); pg_stat_reset_shared ---------------------- @@ -1535,6 +1592,30 @@ SELECT :io_stats_post_reset < :io_stats_pre_reset; t (1 row) +SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS my_io_stats_post_reset + FROM pg_my_stat_io \gset +-- pg_stat_reset_shared() did not reset backend IO stats +SELECT :my_io_stats_pre_reset <= :my_io_stats_post_reset; + ?column? +---------- + t +(1 row) + +-- but pg_stat_reset_single_backend_io_counters() does +SELECT pg_stat_reset_single_backend_io_counters(pg_backend_pid()); + pg_stat_reset_single_backend_io_counters +------------------------------------------ + +(1 row) + +SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS my_io_stats_post_backend_reset + FROM pg_my_stat_io \gset +SELECT :my_io_stats_pre_reset > :my_io_stats_post_backend_reset; + ?column? +---------- + t +(1 row) + -- test BRIN index doesn't block HOT update CREATE TABLE brin_hot ( id integer PRIMARY KEY, diff --git a/src/test/regress/sql/stats.sql b/src/test/regress/sql/stats.sql index 7147cc2f89..497b2b438a 100644 --- a/src/test/regress/sql/stats.sql +++ b/src/test/regress/sql/stats.sql @@ -595,7 +595,7 @@ SELECT pg_stat_get_replication_slot(NULL); SELECT pg_stat_get_subscription_stats(NULL); --- Test that the following operations are tracked in pg_stat_io: +-- Test that the following operations are tracked in pg_[my_]stat_io: -- - reads of target blocks into shared buffers -- - writes of shared buffers to permanent storage -- - extends of relations using shared buffers @@ -607,20 +607,40 @@ SELECT pg_stat_get_subscription_stats(NULL); -- Create a regular table and insert some data to generate IOCONTEXT_NORMAL -- extends. +SELECT pid AS checkpointer_pid FROM pg_stat_activity + WHERE backend_type = 'checkpointer' \gset SELECT sum(extends) AS io_sum_shared_before_extends FROM pg_stat_io WHERE context = 'normal' AND object = 'relation' \gset +SELECT sum(extends) AS my_io_sum_shared_before_extends + FROM pg_my_stat_io WHERE context = 'normal' AND object = 'relation' \gset +SELECT sum(extends) AS backend_io_sum_shared_before_extends + FROM pg_stat_get_backend_io(pg_backend_pid()) + WHERE context = 'normal' AND object = 'relation' \gset SELECT sum(writes) AS writes, sum(fsyncs) AS fsyncs FROM pg_stat_io WHERE object = 'relation' \gset io_sum_shared_before_ +SELECT sum(writes) AS writes, sum(fsyncs) AS fsyncs + FROM pg_my_stat_io + WHERE object = 'relation' \gset my_io_sum_shared_before_ +SELECT sum(writes) AS writes, sum(fsyncs) AS fsyncs + FROM pg_stat_get_backend_io(pg_backend_pid()) + WHERE object = 'relation' \gset backend_io_sum_shared_before_ CREATE TABLE test_io_shared(a int); INSERT INTO test_io_shared SELECT i FROM generate_series(1,100)i; SELECT pg_stat_force_next_flush(); SELECT sum(extends) AS io_sum_shared_after_extends FROM pg_stat_io WHERE context = 'normal' AND object = 'relation' \gset SELECT :io_sum_shared_after_extends > :io_sum_shared_before_extends; +SELECT sum(extends) AS my_io_sum_shared_after_extends + FROM pg_my_stat_io WHERE context = 'normal' AND object = 'relation' \gset +SELECT :my_io_sum_shared_after_extends > :my_io_sum_shared_before_extends; +SELECT sum(extends) AS backend_io_sum_shared_after_extends + FROM pg_stat_get_backend_io(pg_backend_pid()) + WHERE context = 'normal' AND object = 'relation' \gset +SELECT :backend_io_sum_shared_after_extends > :backend_io_sum_shared_before_extends; -- After a checkpoint, there should be some additional IOCONTEXT_NORMAL writes --- and fsyncs. +-- and fsyncs in the global stats (not for the backend). -- See comment above for rationale for two explicit CHECKPOINTs. CHECKPOINT; CHECKPOINT; @@ -631,6 +651,18 @@ SELECT :io_sum_shared_after_writes > :io_sum_shared_before_writes; SELECT current_setting('fsync') = 'off' OR :io_sum_shared_after_fsyncs > :io_sum_shared_before_fsyncs; +SELECT sum(writes) AS writes, sum(fsyncs) AS fsyncs + FROM pg_my_stat_io + WHERE object = 'relation' \gset my_io_sum_shared_after_ +SELECT :my_io_sum_shared_after_writes >= :my_io_sum_shared_before_writes; +SELECT current_setting('fsync') = 'off' + OR (:my_io_sum_shared_after_fsyncs = :my_io_sum_shared_before_fsyncs + AND :my_io_sum_shared_after_fsyncs= 0); + +-- Don't return any rows if querying other backend's stats that are excluded +-- from the per-backend stats collection (like the checkpointer). +SELECT count(1) = 0 FROM pg_stat_get_backend_io(:checkpointer_pid); + -- Change the tablespace so that the table is rewritten directly, then SELECT -- from it to cause it to be read back into shared buffers. SELECT sum(reads) AS io_sum_shared_before_reads @@ -762,10 +794,21 @@ SELECT :io_sum_bulkwrite_strategy_extends_after > :io_sum_bulkwrite_strategy_ext SELECT pg_stat_have_stats('io', 0, 0); SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS io_stats_pre_reset FROM pg_stat_io \gset +SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS my_io_stats_pre_reset + FROM pg_my_stat_io \gset SELECT pg_stat_reset_shared('io'); SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS io_stats_post_reset FROM pg_stat_io \gset SELECT :io_stats_post_reset < :io_stats_pre_reset; +SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS my_io_stats_post_reset + FROM pg_my_stat_io \gset +-- pg_stat_reset_shared() did not reset backend IO stats +SELECT :my_io_stats_pre_reset <= :my_io_stats_post_reset; +-- but pg_stat_reset_single_backend_io_counters() does +SELECT pg_stat_reset_single_backend_io_counters(pg_backend_pid()); +SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS my_io_stats_post_backend_reset + FROM pg_my_stat_io \gset +SELECT :my_io_stats_pre_reset > :my_io_stats_post_backend_reset; -- test BRIN index doesn't block HOT update diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index b54428b38c..ea676361a5 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2120,6 +2120,7 @@ PgFdwSamplingMethod PgFdwScanState PgIfAddrCallback PgStatShared_Archiver +PgStatShared_Backend PgStatShared_BgWriter PgStatShared_Checkpointer PgStatShared_Common @@ -2135,6 +2136,8 @@ PgStatShared_SLRU PgStatShared_Subscription PgStatShared_Wal PgStat_ArchiverStats +PgStat_Backend +PgStat_BackendPendingIO PgStat_BackendSubEntry PgStat_BgWriterStats PgStat_BktypeIO -- 2.34.1 --msbWIkHyd9NSK3Vl-- ^ permalink raw reply [nested|flat] 25+ messages in thread
* [PATCH v7] per backend I/O statistics @ 2024-10-28 12:50 Bertrand Drouvot <[email protected]> 0 siblings, 0 replies; 25+ messages in thread From: Bertrand Drouvot @ 2024-10-28 12:50 UTC (permalink / raw) While pg_stat_io provides cluster-wide I/O statistics, this commit adds the ability to track and display per backend I/O statistics. It adds a new statistics kind and 2 new functions: - pg_stat_reset_single_backend_io_counters() to be able to reset the I/O stats for a given backend pid. - pg_stat_get_backend_io() to retrieve I/O statistics for a given backend pid. The new KIND is named PGSTAT_KIND_PER_BACKEND as it could be used in the future to store other statistics (than the I/O ones) per backend. The new KIND is a variable-numbered one and has an automatic cap on the maximum number of entries (as its hash key contains the proc number). There is no need to write the per backend I/O stats to disk (no point to see stats for backends that do not exist anymore after a re-start), so using "write_to_file = false". Note that per backend I/O statistics are not collected for the checkpointer, the background writer, the startup process and the autovacuum launcher as those are already visible in pg_stat_io and there is only one of those. XXX: Bump catalog version needs to be done. --- doc/src/sgml/config.sgml | 8 +- doc/src/sgml/monitoring.sgml | 37 ++++ src/backend/catalog/system_functions.sql | 2 + src/backend/utils/activity/Makefile | 1 + src/backend/utils/activity/backend_status.c | 4 + src/backend/utils/activity/meson.build | 1 + src/backend/utils/activity/pgstat.c | 19 +- src/backend/utils/activity/pgstat_backend.c | 182 +++++++++++++++++++ src/backend/utils/activity/pgstat_io.c | 25 ++- src/backend/utils/activity/pgstat_relation.c | 2 + src/backend/utils/adt/pgstatfuncs.c | 166 +++++++++++++++++ src/include/catalog/pg_proc.dat | 14 ++ src/include/pgstat.h | 31 +++- src/include/utils/pgstat_internal.h | 14 ++ src/test/regress/expected/stats.out | 72 +++++++- src/test/regress/sql/stats.sql | 38 +++- src/tools/pgindent/typedefs.list | 3 + 17 files changed, 598 insertions(+), 21 deletions(-) 10.0% doc/src/sgml/ 30.9% src/backend/utils/activity/ 21.8% src/backend/utils/adt/ 4.6% src/include/catalog/ 7.1% src/include/ 13.0% src/test/regress/expected/ 11.6% src/test/regress/sql/ diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index e0c8325a39..8afca9b110 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -8403,9 +8403,11 @@ COPY postgres_log FROM '/full/path/to/logfile.csv' WITH csv; displayed in <link linkend="monitoring-pg-stat-database-view"> <structname>pg_stat_database</structname></link>, <link linkend="monitoring-pg-stat-io-view"> - <structname>pg_stat_io</structname></link>, in the output of - <xref linkend="sql-explain"/> when the <literal>BUFFERS</literal> option - is used, in the output of <xref linkend="sql-vacuum"/> when + <structname>pg_stat_io</structname></link>, in the output of the + <link linkend="pg-stat-get-backend-io"> + <function>pg_stat_get_backend_io()</function></link> function, in the + output of <xref linkend="sql-explain"/> when the <literal>BUFFERS</literal> + option is used, in the output of <xref linkend="sql-vacuum"/> when the <literal>VERBOSE</literal> option is used, by autovacuum for auto-vacuums and auto-analyzes, when <xref linkend="guc-log-autovacuum-min-duration"/> is set and by diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml index 840d7f8161..fb13c462d1 100644 --- a/doc/src/sgml/monitoring.sgml +++ b/doc/src/sgml/monitoring.sgml @@ -4790,6 +4790,25 @@ description | Waiting for a newly initialized WAL file to reach durable storage </para></entry> </row> + <row> + <entry id="pg-stat-get-backend-io" role="func_table_entry"><para role="func_signature"> + <indexterm> + <primary>pg_stat_get_backend_io</primary> + </indexterm> + <function>pg_stat_get_backend_io</function> ( <type>integer</type> ) + <returnvalue>setof record</returnvalue> + </para> + <para> + Returns I/O statistics about the backend with the specified + process ID. The output fields are exactly the same as the ones in the + <link linkend="monitoring-pg-stat-io-view"> <structname>pg_stat_io</structname></link> + view. The function does not return I/O statistics for the checkpointer, + the background writer, the startup process and the autovacuum launcher + as they are already visible in the <link linkend="monitoring-pg-stat-io-view"> <structname>pg_stat_io</structname></link> + view and there is only one of those. + </para></entry> + </row> + <row> <entry role="func_table_entry"><para role="func_signature"> <indexterm> @@ -4971,6 +4990,24 @@ description | Waiting for a newly initialized WAL file to reach durable storage </para></entry> </row> + <row> + <entry role="func_table_entry"><para role="func_signature"> + <indexterm> + <primary>pg_stat_reset_single_backend_io_counters</primary> + </indexterm> + <function>pg_stat_reset_single_backend_io_counters</function> ( <type>integer</type> ) + <returnvalue>void</returnvalue> + </para> + <para> + Resets I/O statistics for a single backend with the specified process ID + to zero. + </para> + <para> + This function is restricted to superusers by default, but other users + can be granted EXECUTE to run the function. + </para></entry> + </row> + <row> <entry role="func_table_entry"><para role="func_signature"> <indexterm> diff --git a/src/backend/catalog/system_functions.sql b/src/backend/catalog/system_functions.sql index c51dfca802..1b21b242cd 100644 --- a/src/backend/catalog/system_functions.sql +++ b/src/backend/catalog/system_functions.sql @@ -711,6 +711,8 @@ REVOKE EXECUTE ON FUNCTION pg_stat_reset_single_table_counters(oid) FROM public; REVOKE EXECUTE ON FUNCTION pg_stat_reset_single_function_counters(oid) FROM public; +REVOKE EXECUTE ON FUNCTION pg_stat_reset_single_backend_io_counters(integer) FROM public; + REVOKE EXECUTE ON FUNCTION pg_stat_reset_replication_slot(text) FROM public; REVOKE EXECUTE ON FUNCTION pg_stat_have_stats(text, oid, int8) FROM public; diff --git a/src/backend/utils/activity/Makefile b/src/backend/utils/activity/Makefile index b9fd66ea17..24b64a2742 100644 --- a/src/backend/utils/activity/Makefile +++ b/src/backend/utils/activity/Makefile @@ -20,6 +20,7 @@ OBJS = \ backend_status.o \ pgstat.o \ pgstat_archiver.o \ + pgstat_backend.o \ pgstat_bgwriter.o \ pgstat_checkpointer.o \ pgstat_database.o \ diff --git a/src/backend/utils/activity/backend_status.c b/src/backend/utils/activity/backend_status.c index 22c6dc378c..29779ff99a 100644 --- a/src/backend/utils/activity/backend_status.c +++ b/src/backend/utils/activity/backend_status.c @@ -249,6 +249,10 @@ pgstat_beinit(void) Assert(MyProcNumber >= 0 && MyProcNumber < NumBackendStatSlots); MyBEEntry = &BackendStatusArray[MyProcNumber]; + /* Create the per-backend statistics entry */ + if (pgstat_tracks_per_backend_bktype(MyBackendType)) + pgstat_create_backend_stat(MyProcNumber); + /* Set up a process-exit hook to clean up */ on_shmem_exit(pgstat_beshutdown_hook, 0); } diff --git a/src/backend/utils/activity/meson.build b/src/backend/utils/activity/meson.build index f73c22905c..380d3dd70c 100644 --- a/src/backend/utils/activity/meson.build +++ b/src/backend/utils/activity/meson.build @@ -5,6 +5,7 @@ backend_sources += files( 'backend_status.c', 'pgstat.c', 'pgstat_archiver.c', + 'pgstat_backend.c', 'pgstat_bgwriter.c', 'pgstat_checkpointer.c', 'pgstat_database.c', diff --git a/src/backend/utils/activity/pgstat.c b/src/backend/utils/activity/pgstat.c index 18b7d9b47d..738ee778bb 100644 --- a/src/backend/utils/activity/pgstat.c +++ b/src/backend/utils/activity/pgstat.c @@ -77,6 +77,7 @@ * * Each statistics kind is handled in a dedicated file: * - pgstat_archiver.c + * - pgstat_backend.c * - pgstat_bgwriter.c * - pgstat_checkpointer.c * - pgstat_database.c @@ -358,6 +359,22 @@ static const PgStat_KindInfo pgstat_kind_builtin_infos[PGSTAT_KIND_BUILTIN_SIZE] .reset_timestamp_cb = pgstat_subscription_reset_timestamp_cb, }, + [PGSTAT_KIND_PER_BACKEND] = { + .name = "per-backend", + + .fixed_amount = false, + .write_to_file = false, + + .accessed_across_databases = true, + + .shared_size = sizeof(PgStatShared_Backend), + .shared_data_off = offsetof(PgStatShared_Backend, stats), + .shared_data_len = sizeof(((PgStatShared_Backend *) 0)->stats), + .pending_size = sizeof(PgStat_BackendPendingIO), + + .flush_pending_cb = pgstat_per_backend_flush_cb, + .reset_timestamp_cb = pgstat_backend_reset_timestamp_cb, + }, /* stats for fixed-numbered (mostly 1) objects */ @@ -768,7 +785,7 @@ pgstat_report_stat(bool force) partial_flush = false; - /* flush database / relation / function / ... stats */ + /* flush database / relation / function / backend / ... stats */ partial_flush |= pgstat_flush_pending_entries(nowait); /* flush of fixed-numbered stats */ diff --git a/src/backend/utils/activity/pgstat_backend.c b/src/backend/utils/activity/pgstat_backend.c new file mode 100644 index 0000000000..dc7bf385c2 --- /dev/null +++ b/src/backend/utils/activity/pgstat_backend.c @@ -0,0 +1,182 @@ +/* ------------------------------------------------------------------------- + * + * pgstat_backend.c + * Implementation of per-backend statistics. + * + * This file contains the implementation of per-backend statistics. It is kept + * separate from pgstat.c to enforce the line between the statistics access / + * storage implementation and the details about individual types of statistics. + * + * Copyright (c) 2024, PostgreSQL Global Development Group + * + * IDENTIFICATION + * src/backend/utils/activity/pgstat_backend.c + * ------------------------------------------------------------------------- + */ + +#include "postgres.h" + +#include "utils/pgstat_internal.h" + +/* + * Returns backend's IO stats. + */ +PgStat_Backend * +pgstat_fetch_proc_stat_io(ProcNumber procNumber) +{ + PgStat_Backend *backend_entry; + + backend_entry = (PgStat_Backend *) pgstat_fetch_entry(PGSTAT_KIND_PER_BACKEND, + InvalidOid, procNumber); + + return backend_entry; +} + +/* + * Flush out locally pending backend statistics + * + * If no stats have been recorded, this function returns false. + */ +bool +pgstat_per_backend_flush_cb(PgStat_EntryRef *entry_ref, bool nowait) +{ + PgStatShared_Backend *shbackendioent; + PgStat_BackendPendingIO *pendingent; + PgStat_BktypeIO *bktype_shstats; + + if (!pgstat_lock_entry(entry_ref, nowait)) + return false; + + shbackendioent = (PgStatShared_Backend *) entry_ref->shared_stats; + bktype_shstats = &shbackendioent->stats.stats; + pendingent = (PgStat_BackendPendingIO *) entry_ref->pending; + + for (int io_object = 0; io_object < IOOBJECT_NUM_TYPES; io_object++) + { + for (int io_context = 0; io_context < IOCONTEXT_NUM_TYPES; io_context++) + { + for (int io_op = 0; io_op < IOOP_NUM_TYPES; io_op++) + { + instr_time time; + + bktype_shstats->counts[io_object][io_context][io_op] += + pendingent->counts[io_object][io_context][io_op]; + + time = pendingent->pending_times[io_object][io_context][io_op]; + + bktype_shstats->times[io_object][io_context][io_op] += + INSTR_TIME_GET_MICROSEC(time); + } + } + } + + pgstat_unlock_entry(entry_ref); + + return true; +} + +/* + * Simpler wrapper of pgstat_per_backend_flush_cb() + */ +void +pgstat_flush_per_backend(bool nowait) +{ + if (pgstat_tracks_per_backend_bktype(MyBackendType)) + { + PgStat_EntryRef *entry_ref; + + entry_ref = pgstat_get_entry_ref(PGSTAT_KIND_PER_BACKEND, InvalidOid, + MyProcNumber, false, NULL); + (void) pgstat_per_backend_flush_cb(entry_ref, nowait); + } +} + +/* + * Create the per-backend statistics entry for procnum. + */ +void +pgstat_create_backend_stat(ProcNumber procnum) +{ + PgStat_EntryRef *entry_ref; + PgStatShared_Backend *shstatent; + + entry_ref = pgstat_prep_pending_entry(PGSTAT_KIND_PER_BACKEND, InvalidOid, + procnum, NULL); + + shstatent = (PgStatShared_Backend *) entry_ref->shared_stats; + + /* + * NB: need to accept that there might be stats from an older backend, + * e.g. if we previously used this proc number. + */ + memset(&shstatent->stats, 0, sizeof(shstatent->stats)); +} + +/* + * Find or create a local PgStat_BackendPendingIO entry for procnum. + */ +PgStat_BackendPendingIO * +pgstat_prep_per_backend_pending(ProcNumber procnum) +{ + PgStat_EntryRef *entry_ref; + + entry_ref = pgstat_prep_pending_entry(PGSTAT_KIND_PER_BACKEND, InvalidOid, + procnum, NULL); + + return entry_ref->pending; +} + +/* + * per-backend statistics are not collected for all BackendTypes. + * + * The following BackendTypes do not participate in the per-backend stats + * subsystem: + * - The same and for the same reasons as in pgstat_tracks_io_bktype(). + * - B_BG_WRITER, B_CHECKPOINTER, B_STARTUP and B_AUTOVAC_LAUNCHER because their + * I/O stats are already visible in pg_stat_io and there is only one of those. + * + * Function returns true if BackendType participates in the per-backend stats + * subsystem for IO and false if it does not. + * + * When adding a new BackendType, also consider adding relevant restrictions to + * pgstat_tracks_io_object() and pgstat_tracks_io_op(). + */ +bool +pgstat_tracks_per_backend_bktype(BackendType bktype) +{ + /* + * List every type so that new backend types trigger a warning about + * needing to adjust this switch. + */ + switch (bktype) + { + case B_INVALID: + case B_AUTOVAC_LAUNCHER: + case B_DEAD_END_BACKEND: + case B_ARCHIVER: + case B_LOGGER: + case B_WAL_RECEIVER: + case B_WAL_WRITER: + case B_WAL_SUMMARIZER: + case B_BG_WRITER: + case B_CHECKPOINTER: + case B_STARTUP: + return false; + + case B_AUTOVAC_WORKER: + case B_BACKEND: + case B_BG_WORKER: + case B_STANDALONE_BACKEND: + case B_SLOTSYNC_WORKER: + case B_WAL_SENDER: + return true; + } + + return false; +} + +void +pgstat_backend_reset_timestamp_cb(PgStatShared_Common *header, TimestampTz ts) +{ + ((PgStatShared_Backend *) header)->stats.stat_reset_timestamp = ts; +} diff --git a/src/backend/utils/activity/pgstat_io.c b/src/backend/utils/activity/pgstat_io.c index f9883af2b3..93d34696b9 100644 --- a/src/backend/utils/activity/pgstat_io.c +++ b/src/backend/utils/activity/pgstat_io.c @@ -20,13 +20,7 @@ #include "storage/bufmgr.h" #include "utils/pgstat_internal.h" - -typedef struct PgStat_PendingIO -{ - PgStat_Counter counts[IOOBJECT_NUM_TYPES][IOCONTEXT_NUM_TYPES][IOOP_NUM_TYPES]; - instr_time pending_times[IOOBJECT_NUM_TYPES][IOCONTEXT_NUM_TYPES][IOOP_NUM_TYPES]; -} PgStat_PendingIO; - +typedef PgStat_BackendPendingIO PgStat_PendingIO; static PgStat_PendingIO PendingIOStats; static bool have_iostats = false; @@ -87,6 +81,14 @@ pgstat_count_io_op_n(IOObject io_object, IOContext io_context, IOOp io_op, uint3 Assert((unsigned int) io_op < IOOP_NUM_TYPES); Assert(pgstat_tracks_io_op(MyBackendType, io_object, io_context, io_op)); + if (pgstat_tracks_per_backend_bktype(MyBackendType)) + { + PgStat_PendingIO *entry_ref; + + entry_ref = pgstat_prep_per_backend_pending(MyProcNumber); + entry_ref->counts[io_object][io_context][io_op] += cnt; + } + PendingIOStats.counts[io_object][io_context][io_op] += cnt; have_iostats = true; @@ -148,6 +150,15 @@ pgstat_count_io_op_time(IOObject io_object, IOContext io_context, IOOp io_op, INSTR_TIME_ADD(PendingIOStats.pending_times[io_object][io_context][io_op], io_time); + + if (pgstat_tracks_per_backend_bktype(MyBackendType)) + { + PgStat_PendingIO *entry_ref; + + entry_ref = pgstat_prep_per_backend_pending(MyProcNumber); + INSTR_TIME_ADD(entry_ref->pending_times[io_object][io_context][io_op], + io_time); + } } pgstat_count_io_op_n(io_object, io_context, io_op, cnt); diff --git a/src/backend/utils/activity/pgstat_relation.c b/src/backend/utils/activity/pgstat_relation.c index faba8b64d2..c083c5bf01 100644 --- a/src/backend/utils/activity/pgstat_relation.c +++ b/src/backend/utils/activity/pgstat_relation.c @@ -264,6 +264,7 @@ pgstat_report_vacuum(Oid tableoid, bool shared, * VACUUM command has processed all tables and committed. */ pgstat_flush_io(false); + pgstat_flush_per_backend(false); } /* @@ -350,6 +351,7 @@ pgstat_report_analyze(Relation rel, /* see pgstat_report_vacuum() */ pgstat_flush_io(false); + pgstat_flush_per_backend(false); } /* diff --git a/src/backend/utils/adt/pgstatfuncs.c b/src/backend/utils/adt/pgstatfuncs.c index cdf37403e9..ce4d22e16c 100644 --- a/src/backend/utils/adt/pgstatfuncs.c +++ b/src/backend/utils/adt/pgstatfuncs.c @@ -1474,6 +1474,151 @@ pg_stat_get_io(PG_FUNCTION_ARGS) return (Datum) 0; } +Datum +pg_stat_get_backend_io(PG_FUNCTION_ARGS) +{ + ReturnSetInfo *rsinfo; + PgStat_Backend *backend_stats; + Datum bktype_desc; + PgStat_BktypeIO *bktype_stats; + BackendType bktype; + Datum reset_time; + int num_backends = pgstat_fetch_stat_numbackends(); + int curr_backend; + int pid; + PGPROC *proc; + ProcNumber procNumber; + + InitMaterializedSRF(fcinfo, 0); + rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; + + pid = PG_GETARG_INT32(0); + proc = BackendPidGetProc(pid); + + /* + * Could be an auxiliary process but would not report any stats due to + * pgstat_tracks_per_backend_bktype() anyway. So don't need an extra call + * to AuxiliaryPidGetProc(). + */ + if (!proc) + return (Datum) 0; + + procNumber = GetNumberFromPGProc(proc); + backend_stats = pgstat_fetch_proc_stat_io(procNumber); + + if (!backend_stats) + return (Datum) 0; + + /* just to keep compiler quiet */ + bktype = B_INVALID; + + /* Look for the backend type */ + for (curr_backend = 1; curr_backend <= num_backends; curr_backend++) + { + LocalPgBackendStatus *local_beentry; + PgBackendStatus *beentry; + + /* Get the next one in the list */ + local_beentry = pgstat_get_local_beentry_by_index(curr_backend); + beentry = &local_beentry->backendStatus; + + /* looking for specific PID, ignore all the others */ + if (beentry->st_procpid != pid) + continue; + + bktype = beentry->st_backendType; + break; + } + + bktype_desc = CStringGetTextDatum(GetBackendTypeDesc(bktype)); + bktype_stats = &backend_stats->stats; + reset_time = TimestampTzGetDatum(backend_stats->stat_reset_timestamp); + + /* + * In Assert builds, we can afford an extra loop through all of the + * counters checking that only expected stats are non-zero, since it keeps + * the non-Assert code cleaner. + */ + Assert(pgstat_bktype_io_stats_valid(bktype_stats, bktype)); + + for (int io_obj = 0; io_obj < IOOBJECT_NUM_TYPES; io_obj++) + { + const char *obj_name = pgstat_get_io_object_name(io_obj); + + for (int io_context = 0; io_context < IOCONTEXT_NUM_TYPES; io_context++) + { + const char *context_name = pgstat_get_io_context_name(io_context); + + Datum values[IO_NUM_COLUMNS] = {0}; + bool nulls[IO_NUM_COLUMNS] = {0}; + + /* + * Some combinations of BackendType, IOObject, and IOContext are + * not valid for any type of IOOp. In such cases, omit the entire + * row from the view. + */ + if (!pgstat_tracks_io_object(bktype, io_obj, io_context)) + continue; + + values[IO_COL_BACKEND_TYPE] = bktype_desc; + values[IO_COL_CONTEXT] = CStringGetTextDatum(context_name); + values[IO_COL_OBJECT] = CStringGetTextDatum(obj_name); + if (backend_stats->stat_reset_timestamp != 0) + values[IO_COL_RESET_TIME] = reset_time; + else + nulls[IO_COL_RESET_TIME] = true; + + /* + * Hard-code this to the value of BLCKSZ for now. Future values + * could include XLOG_BLCKSZ, once WAL IO is tracked, and constant + * multipliers, once non-block-oriented IO (e.g. temporary file + * IO) is tracked. + */ + values[IO_COL_CONVERSION] = Int64GetDatum(BLCKSZ); + + for (int io_op = 0; io_op < IOOP_NUM_TYPES; io_op++) + { + int op_idx = pgstat_get_io_op_index(io_op); + int time_idx = pgstat_get_io_time_index(io_op); + + /* + * Some combinations of BackendType and IOOp, of IOContext and + * IOOp, and of IOObject and IOOp are not tracked. Set these + * cells in the view NULL. + */ + if (pgstat_tracks_io_op(bktype, io_obj, io_context, io_op)) + { + PgStat_Counter count = + bktype_stats->counts[io_obj][io_context][io_op]; + + values[op_idx] = Int64GetDatum(count); + } + else + nulls[op_idx] = true; + + /* not every operation is timed */ + if (time_idx == IO_COL_INVALID) + continue; + + if (!nulls[op_idx]) + { + PgStat_Counter time = + bktype_stats->times[io_obj][io_context][io_op]; + + values[time_idx] = Float8GetDatum(pg_stat_us_to_ms(time)); + } + else + nulls[time_idx] = true; + } + + tuplestore_putvalues(rsinfo->setResult, rsinfo->setDesc, + values, nulls); + } + } + + return (Datum) 0; +} + /* * Returns statistics of WAL activity */ @@ -1779,6 +1924,27 @@ pg_stat_reset_single_function_counters(PG_FUNCTION_ARGS) PG_RETURN_VOID(); } +Datum +pg_stat_reset_single_backend_io_counters(PG_FUNCTION_ARGS) +{ + PGPROC *proc; + int backend_pid = PG_GETARG_INT32(0); + + proc = BackendPidGetProc(backend_pid); + + /* + * Could be an auxiliary process but would not report any stats due to + * pgstat_tracks_per_backend_bktype() anyway. So don't need an extra call + * to AuxiliaryPidGetProc(). + */ + if (!proc) + PG_RETURN_VOID(); + + pgstat_reset(PGSTAT_KIND_PER_BACKEND, InvalidOid, GetNumberFromPGProc(proc)); + + PG_RETURN_VOID(); +} + /* Reset SLRU counters (a specific one or all of them). */ Datum pg_stat_reset_slru(PG_FUNCTION_ARGS) diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index 0f22c21723..55dec76795 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -5913,6 +5913,15 @@ proargnames => '{backend_type,object,context,reads,read_time,writes,write_time,writebacks,writeback_time,extends,extend_time,op_bytes,hits,evictions,reuses,fsyncs,fsync_time,stats_reset}', prosrc => 'pg_stat_get_io' }, +{ oid => '8806', descr => 'statistics: per backend IO statistics', + proname => 'pg_stat_get_backend_io', prorows => '5', proretset => 't', + provolatile => 'v', proparallel => 'r', prorettype => 'record', + proargtypes => 'int4', + proallargtypes => '{int4,text,text,text,int8,float8,int8,float8,int8,float8,int8,float8,int8,int8,int8,int8,int8,float8,timestamptz}', + proargmodes => '{i,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o}', + proargnames => '{backend_pid,backend_type,object,context,reads,read_time,writes,write_time,writebacks,writeback_time,extends,extend_time,op_bytes,hits,evictions,reuses,fsyncs,fsync_time,stats_reset}', + prosrc => 'pg_stat_get_backend_io' }, + { oid => '1136', descr => 'statistics: information about WAL activity', proname => 'pg_stat_get_wal', proisstrict => 'f', provolatile => 's', proparallel => 'r', prorettype => 'record', proargtypes => '', @@ -6052,6 +6061,11 @@ proname => 'pg_stat_reset_single_function_counters', provolatile => 'v', prorettype => 'void', proargtypes => 'oid', prosrc => 'pg_stat_reset_single_function_counters' }, +{ oid => '9987', + descr => 'statistics: reset collected IO statistics for a single backend', + proname => 'pg_stat_reset_single_backend_io_counters', provolatile => 'v', + prorettype => 'void', proargtypes => 'int4', + prosrc => 'pg_stat_reset_single_backend_io_counters' }, { oid => '2307', descr => 'statistics: reset collected statistics for a single SLRU', proname => 'pg_stat_reset_slru', proisstrict => 'f', provolatile => 'v', diff --git a/src/include/pgstat.h b/src/include/pgstat.h index 59c28b4aca..b7e8dcb1fd 100644 --- a/src/include/pgstat.h +++ b/src/include/pgstat.h @@ -49,14 +49,15 @@ #define PGSTAT_KIND_FUNCTION 3 /* per-function statistics */ #define PGSTAT_KIND_REPLSLOT 4 /* per-slot statistics */ #define PGSTAT_KIND_SUBSCRIPTION 5 /* per-subscription statistics */ +#define PGSTAT_KIND_PER_BACKEND 6 /* per-backend statistics */ /* stats for fixed-numbered objects */ -#define PGSTAT_KIND_ARCHIVER 6 -#define PGSTAT_KIND_BGWRITER 7 -#define PGSTAT_KIND_CHECKPOINTER 8 -#define PGSTAT_KIND_IO 9 -#define PGSTAT_KIND_SLRU 10 -#define PGSTAT_KIND_WAL 11 +#define PGSTAT_KIND_ARCHIVER 7 +#define PGSTAT_KIND_BGWRITER 8 +#define PGSTAT_KIND_CHECKPOINTER 9 +#define PGSTAT_KIND_IO 10 +#define PGSTAT_KIND_SLRU 11 +#define PGSTAT_KIND_WAL 12 #define PGSTAT_KIND_BUILTIN_MIN PGSTAT_KIND_DATABASE #define PGSTAT_KIND_BUILTIN_MAX PGSTAT_KIND_WAL @@ -347,12 +348,23 @@ typedef struct PgStat_BktypeIO PgStat_Counter times[IOOBJECT_NUM_TYPES][IOCONTEXT_NUM_TYPES][IOOP_NUM_TYPES]; } PgStat_BktypeIO; +typedef struct PgStat_BackendPendingIO +{ + PgStat_Counter counts[IOOBJECT_NUM_TYPES][IOCONTEXT_NUM_TYPES][IOOP_NUM_TYPES]; + instr_time pending_times[IOOBJECT_NUM_TYPES][IOCONTEXT_NUM_TYPES][IOOP_NUM_TYPES]; +} PgStat_BackendPendingIO; + typedef struct PgStat_IO { TimestampTz stat_reset_timestamp; PgStat_BktypeIO stats[BACKEND_NUM_TYPES]; } PgStat_IO; +typedef struct PgStat_Backend +{ + TimestampTz stat_reset_timestamp; + PgStat_BktypeIO stats; +} PgStat_Backend; typedef struct PgStat_StatDBEntry { @@ -534,6 +546,13 @@ extern bool pgstat_have_entry(PgStat_Kind kind, Oid dboid, uint64 objid); extern void pgstat_report_archiver(const char *xlog, bool failed); extern PgStat_ArchiverStats *pgstat_fetch_stat_archiver(void); +/* + * Functions in pgstat_backend.c + */ + +extern PgStat_Backend *pgstat_fetch_proc_stat_io(ProcNumber procNumber); +extern bool pgstat_tracks_per_backend_bktype(BackendType bktype); +extern void pgstat_create_backend_stat(ProcNumber procnum); /* * Functions in pgstat_bgwriter.c diff --git a/src/include/utils/pgstat_internal.h b/src/include/utils/pgstat_internal.h index 7338bc1e28..141e16d1d3 100644 --- a/src/include/utils/pgstat_internal.h +++ b/src/include/utils/pgstat_internal.h @@ -450,6 +450,11 @@ typedef struct PgStatShared_ReplSlot PgStat_StatReplSlotEntry stats; } PgStatShared_ReplSlot; +typedef struct PgStatShared_Backend +{ + PgStatShared_Common header; + PgStat_Backend stats; +} PgStatShared_Backend; /* * Central shared memory entry for the cumulative stats system. @@ -604,6 +609,15 @@ extern void pgstat_archiver_init_shmem_cb(void *stats); extern void pgstat_archiver_reset_all_cb(TimestampTz ts); extern void pgstat_archiver_snapshot_cb(void); +/* + * Functions in pgstat_backend.c + */ + +extern void pgstat_flush_per_backend(bool nowait); + +extern PgStat_BackendPendingIO *pgstat_prep_per_backend_pending(ProcNumber procnum); +extern bool pgstat_per_backend_flush_cb(PgStat_EntryRef *entry_ref, bool nowait); +extern void pgstat_backend_reset_timestamp_cb(PgStatShared_Common *header, TimestampTz ts); /* * Functions in pgstat_bgwriter.c diff --git a/src/test/regress/expected/stats.out b/src/test/regress/expected/stats.out index 56771f83ed..ba8c3cfc88 100644 --- a/src/test/regress/expected/stats.out +++ b/src/test/regress/expected/stats.out @@ -1249,7 +1249,7 @@ SELECT pg_stat_get_subscription_stats(NULL); (1 row) --- Test that the following operations are tracked in pg_stat_io: +-- Test that the following operations are tracked in pg_stat_io and in per-backend stats: -- - reads of target blocks into shared buffers -- - writes of shared buffers to permanent storage -- - extends of relations using shared buffers @@ -1259,11 +1259,19 @@ SELECT pg_stat_get_subscription_stats(NULL); -- be sure of the state of shared buffers at the point the test is run. -- Create a regular table and insert some data to generate IOCONTEXT_NORMAL -- extends. +SELECT pid AS checkpointer_pid FROM pg_stat_activity + WHERE backend_type = 'checkpointer' \gset SELECT sum(extends) AS io_sum_shared_before_extends FROM pg_stat_io WHERE context = 'normal' AND object = 'relation' \gset +SELECT sum(extends) AS my_io_sum_shared_before_extends + FROM pg_stat_get_backend_io(pg_backend_pid()) + WHERE context = 'normal' AND object = 'relation' \gset SELECT sum(writes) AS writes, sum(fsyncs) AS fsyncs FROM pg_stat_io WHERE object = 'relation' \gset io_sum_shared_before_ +SELECT sum(writes) AS writes, sum(fsyncs) AS fsyncs + FROM pg_stat_get_backend_io(pg_backend_pid()) + WHERE object = 'relation' \gset my_io_sum_shared_before_ CREATE TABLE test_io_shared(a int); INSERT INTO test_io_shared SELECT i FROM generate_series(1,100)i; SELECT pg_stat_force_next_flush(); @@ -1280,8 +1288,17 @@ SELECT :io_sum_shared_after_extends > :io_sum_shared_before_extends; t (1 row) +SELECT sum(extends) AS my_io_sum_shared_after_extends + FROM pg_stat_get_backend_io(pg_backend_pid()) + WHERE context = 'normal' AND object = 'relation' \gset +SELECT :my_io_sum_shared_after_extends > :my_io_sum_shared_before_extends; + ?column? +---------- + t +(1 row) + -- After a checkpoint, there should be some additional IOCONTEXT_NORMAL writes --- and fsyncs. +-- and fsyncs in the global stats (not for the backend). -- See comment above for rationale for two explicit CHECKPOINTs. CHECKPOINT; CHECKPOINT; @@ -1301,6 +1318,31 @@ SELECT current_setting('fsync') = 'off' t (1 row) +SELECT sum(writes) AS writes, sum(fsyncs) AS fsyncs + FROM pg_stat_get_backend_io(pg_backend_pid()) + WHERE object = 'relation' \gset my_io_sum_shared_after_ +SELECT :my_io_sum_shared_after_writes >= :my_io_sum_shared_before_writes; + ?column? +---------- + t +(1 row) + +SELECT current_setting('fsync') = 'off' + OR (:my_io_sum_shared_after_fsyncs = :my_io_sum_shared_before_fsyncs + AND :my_io_sum_shared_after_fsyncs= 0); + ?column? +---------- + t +(1 row) + +-- Don't return any rows if querying other backend's stats that are excluded +-- from the per-backend stats collection (like the checkpointer). +SELECT count(1) = 0 FROM pg_stat_get_backend_io(:checkpointer_pid); + ?column? +---------- + t +(1 row) + -- Change the tablespace so that the table is rewritten directly, then SELECT -- from it to cause it to be read back into shared buffers. SELECT sum(reads) AS io_sum_shared_before_reads @@ -1521,6 +1563,8 @@ SELECT pg_stat_have_stats('io', 0, 0); SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS io_stats_pre_reset FROM pg_stat_io \gset +SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS my_io_stats_pre_reset + FROM pg_stat_get_backend_io(pg_backend_pid()) \gset SELECT pg_stat_reset_shared('io'); pg_stat_reset_shared ---------------------- @@ -1535,6 +1579,30 @@ SELECT :io_stats_post_reset < :io_stats_pre_reset; t (1 row) +SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS my_io_stats_post_reset + FROM pg_stat_get_backend_io(pg_backend_pid()) \gset +-- pg_stat_reset_shared() did not reset backend IO stats +SELECT :my_io_stats_pre_reset <= :my_io_stats_post_reset; + ?column? +---------- + t +(1 row) + +-- but pg_stat_reset_single_backend_io_counters() does +SELECT pg_stat_reset_single_backend_io_counters(pg_backend_pid()); + pg_stat_reset_single_backend_io_counters +------------------------------------------ + +(1 row) + +SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS my_io_stats_post_backend_reset + FROM pg_stat_get_backend_io(pg_backend_pid()) \gset +SELECT :my_io_stats_pre_reset > :my_io_stats_post_backend_reset; + ?column? +---------- + t +(1 row) + -- test BRIN index doesn't block HOT update CREATE TABLE brin_hot ( id integer PRIMARY KEY, diff --git a/src/test/regress/sql/stats.sql b/src/test/regress/sql/stats.sql index 7147cc2f89..f17799402f 100644 --- a/src/test/regress/sql/stats.sql +++ b/src/test/regress/sql/stats.sql @@ -595,7 +595,7 @@ SELECT pg_stat_get_replication_slot(NULL); SELECT pg_stat_get_subscription_stats(NULL); --- Test that the following operations are tracked in pg_stat_io: +-- Test that the following operations are tracked in pg_stat_io and in per-backend stats: -- - reads of target blocks into shared buffers -- - writes of shared buffers to permanent storage -- - extends of relations using shared buffers @@ -607,20 +607,32 @@ SELECT pg_stat_get_subscription_stats(NULL); -- Create a regular table and insert some data to generate IOCONTEXT_NORMAL -- extends. +SELECT pid AS checkpointer_pid FROM pg_stat_activity + WHERE backend_type = 'checkpointer' \gset SELECT sum(extends) AS io_sum_shared_before_extends FROM pg_stat_io WHERE context = 'normal' AND object = 'relation' \gset +SELECT sum(extends) AS my_io_sum_shared_before_extends + FROM pg_stat_get_backend_io(pg_backend_pid()) + WHERE context = 'normal' AND object = 'relation' \gset SELECT sum(writes) AS writes, sum(fsyncs) AS fsyncs FROM pg_stat_io WHERE object = 'relation' \gset io_sum_shared_before_ +SELECT sum(writes) AS writes, sum(fsyncs) AS fsyncs + FROM pg_stat_get_backend_io(pg_backend_pid()) + WHERE object = 'relation' \gset my_io_sum_shared_before_ CREATE TABLE test_io_shared(a int); INSERT INTO test_io_shared SELECT i FROM generate_series(1,100)i; SELECT pg_stat_force_next_flush(); SELECT sum(extends) AS io_sum_shared_after_extends FROM pg_stat_io WHERE context = 'normal' AND object = 'relation' \gset SELECT :io_sum_shared_after_extends > :io_sum_shared_before_extends; +SELECT sum(extends) AS my_io_sum_shared_after_extends + FROM pg_stat_get_backend_io(pg_backend_pid()) + WHERE context = 'normal' AND object = 'relation' \gset +SELECT :my_io_sum_shared_after_extends > :my_io_sum_shared_before_extends; -- After a checkpoint, there should be some additional IOCONTEXT_NORMAL writes --- and fsyncs. +-- and fsyncs in the global stats (not for the backend). -- See comment above for rationale for two explicit CHECKPOINTs. CHECKPOINT; CHECKPOINT; @@ -630,6 +642,17 @@ SELECT sum(writes) AS writes, sum(fsyncs) AS fsyncs SELECT :io_sum_shared_after_writes > :io_sum_shared_before_writes; SELECT current_setting('fsync') = 'off' OR :io_sum_shared_after_fsyncs > :io_sum_shared_before_fsyncs; +SELECT sum(writes) AS writes, sum(fsyncs) AS fsyncs + FROM pg_stat_get_backend_io(pg_backend_pid()) + WHERE object = 'relation' \gset my_io_sum_shared_after_ +SELECT :my_io_sum_shared_after_writes >= :my_io_sum_shared_before_writes; +SELECT current_setting('fsync') = 'off' + OR (:my_io_sum_shared_after_fsyncs = :my_io_sum_shared_before_fsyncs + AND :my_io_sum_shared_after_fsyncs= 0); + +-- Don't return any rows if querying other backend's stats that are excluded +-- from the per-backend stats collection (like the checkpointer). +SELECT count(1) = 0 FROM pg_stat_get_backend_io(:checkpointer_pid); -- Change the tablespace so that the table is rewritten directly, then SELECT -- from it to cause it to be read back into shared buffers. @@ -762,10 +785,21 @@ SELECT :io_sum_bulkwrite_strategy_extends_after > :io_sum_bulkwrite_strategy_ext SELECT pg_stat_have_stats('io', 0, 0); SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS io_stats_pre_reset FROM pg_stat_io \gset +SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS my_io_stats_pre_reset + FROM pg_stat_get_backend_io(pg_backend_pid()) \gset SELECT pg_stat_reset_shared('io'); SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS io_stats_post_reset FROM pg_stat_io \gset SELECT :io_stats_post_reset < :io_stats_pre_reset; +SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS my_io_stats_post_reset + FROM pg_stat_get_backend_io(pg_backend_pid()) \gset +-- pg_stat_reset_shared() did not reset backend IO stats +SELECT :my_io_stats_pre_reset <= :my_io_stats_post_reset; +-- but pg_stat_reset_single_backend_io_counters() does +SELECT pg_stat_reset_single_backend_io_counters(pg_backend_pid()); +SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS my_io_stats_post_backend_reset + FROM pg_stat_get_backend_io(pg_backend_pid()) \gset +SELECT :my_io_stats_pre_reset > :my_io_stats_post_backend_reset; -- test BRIN index doesn't block HOT update diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index ce33e55bf1..398dd92527 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2121,6 +2121,7 @@ PgFdwSamplingMethod PgFdwScanState PgIfAddrCallback PgStatShared_Archiver +PgStatShared_Backend PgStatShared_BgWriter PgStatShared_Checkpointer PgStatShared_Common @@ -2136,6 +2137,8 @@ PgStatShared_SLRU PgStatShared_Subscription PgStatShared_Wal PgStat_ArchiverStats +PgStat_Backend +PgStat_BackendPendingIO PgStat_BackendSubEntry PgStat_BgWriterStats PgStat_BktypeIO -- 2.34.1 --PnfPtg2CPUf6YsJo-- ^ permalink raw reply [nested|flat] 25+ messages in thread
* [PATCH v8] per backend I/O statistics @ 2024-10-28 12:50 Bertrand Drouvot <[email protected]> 0 siblings, 0 replies; 25+ messages in thread From: Bertrand Drouvot @ 2024-10-28 12:50 UTC (permalink / raw) While pg_stat_io provides cluster-wide I/O statistics, this commit adds the ability to track and display per backend I/O statistics. It adds a new statistics kind and 2 new functions: - pg_stat_reset_backend_stats() to be able to reset the stats for a given backend pid. - pg_stat_get_backend_io() to retrieve I/O statistics for a given backend pid. The new KIND is named PGSTAT_KIND_BACKEND as it could be used in the future to store other statistics (than the I/O ones) per backend. The new KIND is a variable-numbered one and has an automatic cap on the maximum number of entries (as its hash key contains the proc number). There is no need to write the per backend I/O stats to disk (no point to see stats for backends that do not exist anymore after a re-start), so using "write_to_file = false". Note that per backend I/O statistics are not collected for the checkpointer, the background writer, the startup process and the autovacuum launcher as those are already visible in pg_stat_io and there is only one of those. XXX: Bump catalog version needs to be done. --- doc/src/sgml/config.sgml | 8 +- doc/src/sgml/monitoring.sgml | 37 ++++ src/backend/catalog/system_functions.sql | 2 + src/backend/utils/activity/Makefile | 1 + src/backend/utils/activity/backend_status.c | 4 + src/backend/utils/activity/meson.build | 1 + src/backend/utils/activity/pgstat.c | 22 ++- src/backend/utils/activity/pgstat_backend.c | 182 +++++++++++++++++++ src/backend/utils/activity/pgstat_io.c | 25 ++- src/backend/utils/activity/pgstat_relation.c | 2 + src/backend/utils/adt/pgstatfuncs.c | 169 +++++++++++++++++ src/include/catalog/pg_proc.dat | 14 ++ src/include/pgstat.h | 31 +++- src/include/utils/pgstat_internal.h | 14 ++ src/test/regress/expected/stats.out | 72 +++++++- src/test/regress/sql/stats.sql | 38 +++- src/tools/pgindent/typedefs.list | 3 + 17 files changed, 604 insertions(+), 21 deletions(-) 9.9% doc/src/sgml/ 31.2% src/backend/utils/activity/ 22.1% src/backend/utils/adt/ 4.4% src/include/catalog/ 7.1% src/include/ 12.9% src/test/regress/expected/ 11.5% src/test/regress/sql/ diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index e0c8325a39..8afca9b110 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -8403,9 +8403,11 @@ COPY postgres_log FROM '/full/path/to/logfile.csv' WITH csv; displayed in <link linkend="monitoring-pg-stat-database-view"> <structname>pg_stat_database</structname></link>, <link linkend="monitoring-pg-stat-io-view"> - <structname>pg_stat_io</structname></link>, in the output of - <xref linkend="sql-explain"/> when the <literal>BUFFERS</literal> option - is used, in the output of <xref linkend="sql-vacuum"/> when + <structname>pg_stat_io</structname></link>, in the output of the + <link linkend="pg-stat-get-backend-io"> + <function>pg_stat_get_backend_io()</function></link> function, in the + output of <xref linkend="sql-explain"/> when the <literal>BUFFERS</literal> + option is used, in the output of <xref linkend="sql-vacuum"/> when the <literal>VERBOSE</literal> option is used, by autovacuum for auto-vacuums and auto-analyzes, when <xref linkend="guc-log-autovacuum-min-duration"/> is set and by diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml index 840d7f8161..4a9464da61 100644 --- a/doc/src/sgml/monitoring.sgml +++ b/doc/src/sgml/monitoring.sgml @@ -4790,6 +4790,25 @@ description | Waiting for a newly initialized WAL file to reach durable storage </para></entry> </row> + <row> + <entry id="pg-stat-get-backend-io" role="func_table_entry"><para role="func_signature"> + <indexterm> + <primary>pg_stat_get_backend_io</primary> + </indexterm> + <function>pg_stat_get_backend_io</function> ( <type>integer</type> ) + <returnvalue>setof record</returnvalue> + </para> + <para> + Returns I/O statistics about the backend with the specified + process ID. The output fields are exactly the same as the ones in the + <link linkend="monitoring-pg-stat-io-view"> <structname>pg_stat_io</structname></link> + view. The function does not return I/O statistics for the checkpointer, + the background writer, the startup process and the autovacuum launcher + as they are already visible in the <link linkend="monitoring-pg-stat-io-view"> <structname>pg_stat_io</structname></link> + view and there is only one of those. + </para></entry> + </row> + <row> <entry role="func_table_entry"><para role="func_signature"> <indexterm> @@ -4971,6 +4990,24 @@ description | Waiting for a newly initialized WAL file to reach durable storage </para></entry> </row> + <row> + <entry role="func_table_entry"><para role="func_signature"> + <indexterm> + <primary>pg_stat_reset_backend_stats</primary> + </indexterm> + <function>pg_stat_reset_backend_stats</function> ( <type>integer</type> ) + <returnvalue>void</returnvalue> + </para> + <para> + Resets statistics for a single backend with the specified process ID + to zero. + </para> + <para> + This function is restricted to superusers by default, but other users + can be granted EXECUTE to run the function. + </para></entry> + </row> + <row> <entry role="func_table_entry"><para role="func_signature"> <indexterm> diff --git a/src/backend/catalog/system_functions.sql b/src/backend/catalog/system_functions.sql index c51dfca802..14eb99cd47 100644 --- a/src/backend/catalog/system_functions.sql +++ b/src/backend/catalog/system_functions.sql @@ -711,6 +711,8 @@ REVOKE EXECUTE ON FUNCTION pg_stat_reset_single_table_counters(oid) FROM public; REVOKE EXECUTE ON FUNCTION pg_stat_reset_single_function_counters(oid) FROM public; +REVOKE EXECUTE ON FUNCTION pg_stat_reset_backend_stats(integer) FROM public; + REVOKE EXECUTE ON FUNCTION pg_stat_reset_replication_slot(text) FROM public; REVOKE EXECUTE ON FUNCTION pg_stat_have_stats(text, oid, int8) FROM public; diff --git a/src/backend/utils/activity/Makefile b/src/backend/utils/activity/Makefile index b9fd66ea17..24b64a2742 100644 --- a/src/backend/utils/activity/Makefile +++ b/src/backend/utils/activity/Makefile @@ -20,6 +20,7 @@ OBJS = \ backend_status.o \ pgstat.o \ pgstat_archiver.o \ + pgstat_backend.o \ pgstat_bgwriter.o \ pgstat_checkpointer.o \ pgstat_database.o \ diff --git a/src/backend/utils/activity/backend_status.c b/src/backend/utils/activity/backend_status.c index 22c6dc378c..7f74011a00 100644 --- a/src/backend/utils/activity/backend_status.c +++ b/src/backend/utils/activity/backend_status.c @@ -249,6 +249,10 @@ pgstat_beinit(void) Assert(MyProcNumber >= 0 && MyProcNumber < NumBackendStatSlots); MyBEEntry = &BackendStatusArray[MyProcNumber]; + /* Create the backend statistics entry */ + if (pgstat_tracks_backend_bktype(MyBackendType)) + pgstat_create_backend_stat(MyProcNumber); + /* Set up a process-exit hook to clean up */ on_shmem_exit(pgstat_beshutdown_hook, 0); } diff --git a/src/backend/utils/activity/meson.build b/src/backend/utils/activity/meson.build index f73c22905c..380d3dd70c 100644 --- a/src/backend/utils/activity/meson.build +++ b/src/backend/utils/activity/meson.build @@ -5,6 +5,7 @@ backend_sources += files( 'backend_status.c', 'pgstat.c', 'pgstat_archiver.c', + 'pgstat_backend.c', 'pgstat_bgwriter.c', 'pgstat_checkpointer.c', 'pgstat_database.c', diff --git a/src/backend/utils/activity/pgstat.c b/src/backend/utils/activity/pgstat.c index 18b7d9b47d..563f2443f8 100644 --- a/src/backend/utils/activity/pgstat.c +++ b/src/backend/utils/activity/pgstat.c @@ -77,6 +77,7 @@ * * Each statistics kind is handled in a dedicated file: * - pgstat_archiver.c + * - pgstat_backend.c * - pgstat_bgwriter.c * - pgstat_checkpointer.c * - pgstat_database.c @@ -358,6 +359,22 @@ static const PgStat_KindInfo pgstat_kind_builtin_infos[PGSTAT_KIND_BUILTIN_SIZE] .reset_timestamp_cb = pgstat_subscription_reset_timestamp_cb, }, + [PGSTAT_KIND_BACKEND] = { + .name = "backend", + + .fixed_amount = false, + .write_to_file = false, + + .accessed_across_databases = true, + + .shared_size = sizeof(PgStatShared_Backend), + .shared_data_off = offsetof(PgStatShared_Backend, stats), + .shared_data_len = sizeof(((PgStatShared_Backend *) 0)->stats), + .pending_size = sizeof(PgStat_BackendPendingIO), + + .flush_pending_cb = pgstat_backend_flush_cb, + .reset_timestamp_cb = pgstat_backend_reset_timestamp_cb, + }, /* stats for fixed-numbered (mostly 1) objects */ @@ -602,6 +619,9 @@ pgstat_shutdown_hook(int code, Datum arg) Assert(dlist_is_empty(&pgStatPending)); dlist_init(&pgStatPending); + /* drop the backend stats entry */ + pgstat_drop_entry(PGSTAT_KIND_BACKEND, InvalidOid, MyProcNumber); + pgstat_detach_shmem(); #ifdef USE_ASSERT_CHECKING @@ -768,7 +788,7 @@ pgstat_report_stat(bool force) partial_flush = false; - /* flush database / relation / function / ... stats */ + /* flush database / relation / function / backend / ... stats */ partial_flush |= pgstat_flush_pending_entries(nowait); /* flush of fixed-numbered stats */ diff --git a/src/backend/utils/activity/pgstat_backend.c b/src/backend/utils/activity/pgstat_backend.c new file mode 100644 index 0000000000..e2d83024c2 --- /dev/null +++ b/src/backend/utils/activity/pgstat_backend.c @@ -0,0 +1,182 @@ +/* ------------------------------------------------------------------------- + * + * pgstat_backend.c + * Implementation of backend statistics. + * + * This file contains the implementation of backend statistics. It is kept + * separate from pgstat.c to enforce the line between the statistics access / + * storage implementation and the details about individual types of statistics. + * + * Copyright (c) 2024, PostgreSQL Global Development Group + * + * IDENTIFICATION + * src/backend/utils/activity/pgstat_backend.c + * ------------------------------------------------------------------------- + */ + +#include "postgres.h" + +#include "utils/pgstat_internal.h" + +/* + * Returns backend's IO stats. + */ +PgStat_Backend * +pgstat_fetch_proc_stat_io(ProcNumber procNumber) +{ + PgStat_Backend *backend_entry; + + backend_entry = (PgStat_Backend *) pgstat_fetch_entry(PGSTAT_KIND_BACKEND, + InvalidOid, procNumber); + + return backend_entry; +} + +/* + * Flush out locally pending backend statistics + * + * If no stats have been recorded, this function returns false. + */ +bool +pgstat_backend_flush_cb(PgStat_EntryRef *entry_ref, bool nowait) +{ + PgStatShared_Backend *shbackendioent; + PgStat_BackendPendingIO *pendingent; + PgStat_BktypeIO *bktype_shstats; + + if (!pgstat_lock_entry(entry_ref, nowait)) + return false; + + shbackendioent = (PgStatShared_Backend *) entry_ref->shared_stats; + bktype_shstats = &shbackendioent->stats.stats; + pendingent = (PgStat_BackendPendingIO *) entry_ref->pending; + + for (int io_object = 0; io_object < IOOBJECT_NUM_TYPES; io_object++) + { + for (int io_context = 0; io_context < IOCONTEXT_NUM_TYPES; io_context++) + { + for (int io_op = 0; io_op < IOOP_NUM_TYPES; io_op++) + { + instr_time time; + + bktype_shstats->counts[io_object][io_context][io_op] += + pendingent->counts[io_object][io_context][io_op]; + + time = pendingent->pending_times[io_object][io_context][io_op]; + + bktype_shstats->times[io_object][io_context][io_op] += + INSTR_TIME_GET_MICROSEC(time); + } + } + } + + pgstat_unlock_entry(entry_ref); + + return true; +} + +/* + * Simpler wrapper of pgstat_backend_flush_cb() + */ +void +pgstat_flush_backend(bool nowait) +{ + if (pgstat_tracks_backend_bktype(MyBackendType)) + { + PgStat_EntryRef *entry_ref; + + entry_ref = pgstat_get_entry_ref(PGSTAT_KIND_BACKEND, InvalidOid, + MyProcNumber, false, NULL); + (void) pgstat_backend_flush_cb(entry_ref, nowait); + } +} + +/* + * Create the backend statistics entry for procnum. + */ +void +pgstat_create_backend_stat(ProcNumber procnum) +{ + PgStat_EntryRef *entry_ref; + PgStatShared_Backend *shstatent; + + entry_ref = pgstat_prep_pending_entry(PGSTAT_KIND_BACKEND, InvalidOid, + procnum, NULL); + + shstatent = (PgStatShared_Backend *) entry_ref->shared_stats; + + /* + * NB: need to accept that there might be stats from an older backend, + * e.g. if we previously used this proc number. + */ + memset(&shstatent->stats, 0, sizeof(shstatent->stats)); +} + +/* + * Find or create a local PgStat_BackendPendingIO entry for procnum. + */ +PgStat_BackendPendingIO * +pgstat_prep_backend_pending(ProcNumber procnum) +{ + PgStat_EntryRef *entry_ref; + + entry_ref = pgstat_prep_pending_entry(PGSTAT_KIND_BACKEND, InvalidOid, + procnum, NULL); + + return entry_ref->pending; +} + +/* + * Backend statistics are not collected for all BackendTypes. + * + * The following BackendTypes do not participate in the backend stats + * subsystem: + * - The same and for the same reasons as in pgstat_tracks_io_bktype(). + * - B_BG_WRITER, B_CHECKPOINTER, B_STARTUP and B_AUTOVAC_LAUNCHER because their + * I/O stats are already visible in pg_stat_io and there is only one of those. + * + * Function returns true if BackendType participates in the backend stats + * subsystem for IO and false if it does not. + * + * When adding a new BackendType, also consider adding relevant restrictions to + * pgstat_tracks_io_object() and pgstat_tracks_io_op(). + */ +bool +pgstat_tracks_backend_bktype(BackendType bktype) +{ + /* + * List every type so that new backend types trigger a warning about + * needing to adjust this switch. + */ + switch (bktype) + { + case B_INVALID: + case B_AUTOVAC_LAUNCHER: + case B_DEAD_END_BACKEND: + case B_ARCHIVER: + case B_LOGGER: + case B_WAL_RECEIVER: + case B_WAL_WRITER: + case B_WAL_SUMMARIZER: + case B_BG_WRITER: + case B_CHECKPOINTER: + case B_STARTUP: + return false; + + case B_AUTOVAC_WORKER: + case B_BACKEND: + case B_BG_WORKER: + case B_STANDALONE_BACKEND: + case B_SLOTSYNC_WORKER: + case B_WAL_SENDER: + return true; + } + + return false; +} + +void +pgstat_backend_reset_timestamp_cb(PgStatShared_Common *header, TimestampTz ts) +{ + ((PgStatShared_Backend *) header)->stats.stat_reset_timestamp = ts; +} diff --git a/src/backend/utils/activity/pgstat_io.c b/src/backend/utils/activity/pgstat_io.c index f9883af2b3..e0c206a453 100644 --- a/src/backend/utils/activity/pgstat_io.c +++ b/src/backend/utils/activity/pgstat_io.c @@ -20,13 +20,7 @@ #include "storage/bufmgr.h" #include "utils/pgstat_internal.h" - -typedef struct PgStat_PendingIO -{ - PgStat_Counter counts[IOOBJECT_NUM_TYPES][IOCONTEXT_NUM_TYPES][IOOP_NUM_TYPES]; - instr_time pending_times[IOOBJECT_NUM_TYPES][IOCONTEXT_NUM_TYPES][IOOP_NUM_TYPES]; -} PgStat_PendingIO; - +typedef PgStat_BackendPendingIO PgStat_PendingIO; static PgStat_PendingIO PendingIOStats; static bool have_iostats = false; @@ -87,6 +81,14 @@ pgstat_count_io_op_n(IOObject io_object, IOContext io_context, IOOp io_op, uint3 Assert((unsigned int) io_op < IOOP_NUM_TYPES); Assert(pgstat_tracks_io_op(MyBackendType, io_object, io_context, io_op)); + if (pgstat_tracks_backend_bktype(MyBackendType)) + { + PgStat_PendingIO *entry_ref; + + entry_ref = pgstat_prep_backend_pending(MyProcNumber); + entry_ref->counts[io_object][io_context][io_op] += cnt; + } + PendingIOStats.counts[io_object][io_context][io_op] += cnt; have_iostats = true; @@ -148,6 +150,15 @@ pgstat_count_io_op_time(IOObject io_object, IOContext io_context, IOOp io_op, INSTR_TIME_ADD(PendingIOStats.pending_times[io_object][io_context][io_op], io_time); + + if (pgstat_tracks_backend_bktype(MyBackendType)) + { + PgStat_PendingIO *entry_ref; + + entry_ref = pgstat_prep_backend_pending(MyProcNumber); + INSTR_TIME_ADD(entry_ref->pending_times[io_object][io_context][io_op], + io_time); + } } pgstat_count_io_op_n(io_object, io_context, io_op, cnt); diff --git a/src/backend/utils/activity/pgstat_relation.c b/src/backend/utils/activity/pgstat_relation.c index faba8b64d2..85e65557bb 100644 --- a/src/backend/utils/activity/pgstat_relation.c +++ b/src/backend/utils/activity/pgstat_relation.c @@ -264,6 +264,7 @@ pgstat_report_vacuum(Oid tableoid, bool shared, * VACUUM command has processed all tables and committed. */ pgstat_flush_io(false); + pgstat_flush_backend(false); } /* @@ -350,6 +351,7 @@ pgstat_report_analyze(Relation rel, /* see pgstat_report_vacuum() */ pgstat_flush_io(false); + pgstat_flush_backend(false); } /* diff --git a/src/backend/utils/adt/pgstatfuncs.c b/src/backend/utils/adt/pgstatfuncs.c index cdf37403e9..b939551d36 100644 --- a/src/backend/utils/adt/pgstatfuncs.c +++ b/src/backend/utils/adt/pgstatfuncs.c @@ -1474,6 +1474,154 @@ pg_stat_get_io(PG_FUNCTION_ARGS) return (Datum) 0; } +Datum +pg_stat_get_backend_io(PG_FUNCTION_ARGS) +{ + ReturnSetInfo *rsinfo; + PgStat_Backend *backend_stats; + Datum bktype_desc; + PgStat_BktypeIO *bktype_stats; + BackendType bktype; + Datum reset_time; + int num_backends = pgstat_fetch_stat_numbackends(); + int curr_backend; + int pid; + PGPROC *proc; + ProcNumber procNumber; + + InitMaterializedSRF(fcinfo, 0); + rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; + + pid = PG_GETARG_INT32(0); + proc = BackendPidGetProc(pid); + + /* + * Could be an auxiliary process but would not report any stats due to + * pgstat_tracks_backend_bktype() anyway. So don't need an extra call to + * AuxiliaryPidGetProc(). + */ + if (!proc) + return (Datum) 0; + + procNumber = GetNumberFromPGProc(proc); + backend_stats = pgstat_fetch_proc_stat_io(procNumber); + + if (!backend_stats) + return (Datum) 0; + + bktype = B_INVALID; + + /* Look for the backend type */ + for (curr_backend = 1; curr_backend <= num_backends; curr_backend++) + { + LocalPgBackendStatus *local_beentry; + PgBackendStatus *beentry; + + /* Get the next one in the list */ + local_beentry = pgstat_get_local_beentry_by_index(curr_backend); + beentry = &local_beentry->backendStatus; + + /* Looking for specific PID, ignore all the others */ + if (beentry->st_procpid != pid) + continue; + + bktype = beentry->st_backendType; + break; + } + + /* Backend is gone */ + if (bktype == B_INVALID) + return (Datum) 0; + + bktype_desc = CStringGetTextDatum(GetBackendTypeDesc(bktype)); + bktype_stats = &backend_stats->stats; + reset_time = TimestampTzGetDatum(backend_stats->stat_reset_timestamp); + + /* + * In Assert builds, we can afford an extra loop through all of the + * counters checking that only expected stats are non-zero, since it keeps + * the non-Assert code cleaner. + */ + Assert(pgstat_bktype_io_stats_valid(bktype_stats, bktype)); + + for (int io_obj = 0; io_obj < IOOBJECT_NUM_TYPES; io_obj++) + { + const char *obj_name = pgstat_get_io_object_name(io_obj); + + for (int io_context = 0; io_context < IOCONTEXT_NUM_TYPES; io_context++) + { + const char *context_name = pgstat_get_io_context_name(io_context); + + Datum values[IO_NUM_COLUMNS] = {0}; + bool nulls[IO_NUM_COLUMNS] = {0}; + + /* + * Some combinations of BackendType, IOObject, and IOContext are + * not valid for any type of IOOp. In such cases, omit the entire + * row from the view. + */ + if (!pgstat_tracks_io_object(bktype, io_obj, io_context)) + continue; + + values[IO_COL_BACKEND_TYPE] = bktype_desc; + values[IO_COL_CONTEXT] = CStringGetTextDatum(context_name); + values[IO_COL_OBJECT] = CStringGetTextDatum(obj_name); + if (backend_stats->stat_reset_timestamp != 0) + values[IO_COL_RESET_TIME] = reset_time; + else + nulls[IO_COL_RESET_TIME] = true; + + /* + * Hard-code this to the value of BLCKSZ for now. Future values + * could include XLOG_BLCKSZ, once WAL IO is tracked, and constant + * multipliers, once non-block-oriented IO (e.g. temporary file + * IO) is tracked. + */ + values[IO_COL_CONVERSION] = Int64GetDatum(BLCKSZ); + + for (int io_op = 0; io_op < IOOP_NUM_TYPES; io_op++) + { + int op_idx = pgstat_get_io_op_index(io_op); + int time_idx = pgstat_get_io_time_index(io_op); + + /* + * Some combinations of BackendType and IOOp, of IOContext and + * IOOp, and of IOObject and IOOp are not tracked. Set these + * cells in the view NULL. + */ + if (pgstat_tracks_io_op(bktype, io_obj, io_context, io_op)) + { + PgStat_Counter count = + bktype_stats->counts[io_obj][io_context][io_op]; + + values[op_idx] = Int64GetDatum(count); + } + else + nulls[op_idx] = true; + + /* not every operation is timed */ + if (time_idx == IO_COL_INVALID) + continue; + + if (!nulls[op_idx]) + { + PgStat_Counter time = + bktype_stats->times[io_obj][io_context][io_op]; + + values[time_idx] = Float8GetDatum(pg_stat_us_to_ms(time)); + } + else + nulls[time_idx] = true; + } + + tuplestore_putvalues(rsinfo->setResult, rsinfo->setDesc, + values, nulls); + } + } + + return (Datum) 0; +} + /* * Returns statistics of WAL activity */ @@ -1779,6 +1927,27 @@ pg_stat_reset_single_function_counters(PG_FUNCTION_ARGS) PG_RETURN_VOID(); } +Datum +pg_stat_reset_backend_stats(PG_FUNCTION_ARGS) +{ + PGPROC *proc; + int backend_pid = PG_GETARG_INT32(0); + + proc = BackendPidGetProc(backend_pid); + + /* + * Could be an auxiliary process but would not report any stats due to + * pgstat_tracks_backend_bktype() anyway. So don't need an extra call to + * AuxiliaryPidGetProc(). + */ + if (!proc) + PG_RETURN_VOID(); + + pgstat_reset(PGSTAT_KIND_BACKEND, InvalidOid, GetNumberFromPGProc(proc)); + + PG_RETURN_VOID(); +} + /* Reset SLRU counters (a specific one or all of them). */ Datum pg_stat_reset_slru(PG_FUNCTION_ARGS) diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index 0f22c21723..437157ffa3 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -5913,6 +5913,15 @@ proargnames => '{backend_type,object,context,reads,read_time,writes,write_time,writebacks,writeback_time,extends,extend_time,op_bytes,hits,evictions,reuses,fsyncs,fsync_time,stats_reset}', prosrc => 'pg_stat_get_io' }, +{ oid => '8806', descr => 'statistics: backend IO statistics', + proname => 'pg_stat_get_backend_io', prorows => '5', proretset => 't', + provolatile => 'v', proparallel => 'r', prorettype => 'record', + proargtypes => 'int4', + proallargtypes => '{int4,text,text,text,int8,float8,int8,float8,int8,float8,int8,float8,int8,int8,int8,int8,int8,float8,timestamptz}', + proargmodes => '{i,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o}', + proargnames => '{backend_pid,backend_type,object,context,reads,read_time,writes,write_time,writebacks,writeback_time,extends,extend_time,op_bytes,hits,evictions,reuses,fsyncs,fsync_time,stats_reset}', + prosrc => 'pg_stat_get_backend_io' }, + { oid => '1136', descr => 'statistics: information about WAL activity', proname => 'pg_stat_get_wal', proisstrict => 'f', provolatile => 's', proparallel => 'r', prorettype => 'record', proargtypes => '', @@ -6052,6 +6061,11 @@ proname => 'pg_stat_reset_single_function_counters', provolatile => 'v', prorettype => 'void', proargtypes => 'oid', prosrc => 'pg_stat_reset_single_function_counters' }, +{ oid => '9987', + descr => 'statistics: reset statistics for a single backend', + proname => 'pg_stat_reset_backend_stats', provolatile => 'v', + prorettype => 'void', proargtypes => 'int4', + prosrc => 'pg_stat_reset_backend_stats' }, { oid => '2307', descr => 'statistics: reset collected statistics for a single SLRU', proname => 'pg_stat_reset_slru', proisstrict => 'f', provolatile => 'v', diff --git a/src/include/pgstat.h b/src/include/pgstat.h index ebfeef2f46..479773cfd2 100644 --- a/src/include/pgstat.h +++ b/src/include/pgstat.h @@ -49,14 +49,15 @@ #define PGSTAT_KIND_FUNCTION 3 /* per-function statistics */ #define PGSTAT_KIND_REPLSLOT 4 /* per-slot statistics */ #define PGSTAT_KIND_SUBSCRIPTION 5 /* per-subscription statistics */ +#define PGSTAT_KIND_BACKEND 6 /* per-backend statistics */ /* stats for fixed-numbered objects */ -#define PGSTAT_KIND_ARCHIVER 6 -#define PGSTAT_KIND_BGWRITER 7 -#define PGSTAT_KIND_CHECKPOINTER 8 -#define PGSTAT_KIND_IO 9 -#define PGSTAT_KIND_SLRU 10 -#define PGSTAT_KIND_WAL 11 +#define PGSTAT_KIND_ARCHIVER 7 +#define PGSTAT_KIND_BGWRITER 8 +#define PGSTAT_KIND_CHECKPOINTER 9 +#define PGSTAT_KIND_IO 10 +#define PGSTAT_KIND_SLRU 11 +#define PGSTAT_KIND_WAL 12 #define PGSTAT_KIND_BUILTIN_MIN PGSTAT_KIND_DATABASE #define PGSTAT_KIND_BUILTIN_MAX PGSTAT_KIND_WAL @@ -362,12 +363,23 @@ typedef struct PgStat_BktypeIO PgStat_Counter times[IOOBJECT_NUM_TYPES][IOCONTEXT_NUM_TYPES][IOOP_NUM_TYPES]; } PgStat_BktypeIO; +typedef struct PgStat_BackendPendingIO +{ + PgStat_Counter counts[IOOBJECT_NUM_TYPES][IOCONTEXT_NUM_TYPES][IOOP_NUM_TYPES]; + instr_time pending_times[IOOBJECT_NUM_TYPES][IOCONTEXT_NUM_TYPES][IOOP_NUM_TYPES]; +} PgStat_BackendPendingIO; + typedef struct PgStat_IO { TimestampTz stat_reset_timestamp; PgStat_BktypeIO stats[BACKEND_NUM_TYPES]; } PgStat_IO; +typedef struct PgStat_Backend +{ + TimestampTz stat_reset_timestamp; + PgStat_BktypeIO stats; +} PgStat_Backend; typedef struct PgStat_StatDBEntry { @@ -549,6 +561,13 @@ extern bool pgstat_have_entry(PgStat_Kind kind, Oid dboid, uint64 objid); extern void pgstat_report_archiver(const char *xlog, bool failed); extern PgStat_ArchiverStats *pgstat_fetch_stat_archiver(void); +/* + * Functions in pgstat_backend.c + */ + +extern PgStat_Backend *pgstat_fetch_proc_stat_io(ProcNumber procNumber); +extern bool pgstat_tracks_backend_bktype(BackendType bktype); +extern void pgstat_create_backend_stat(ProcNumber procnum); /* * Functions in pgstat_bgwriter.c diff --git a/src/include/utils/pgstat_internal.h b/src/include/utils/pgstat_internal.h index 7338bc1e28..811ed9b005 100644 --- a/src/include/utils/pgstat_internal.h +++ b/src/include/utils/pgstat_internal.h @@ -450,6 +450,11 @@ typedef struct PgStatShared_ReplSlot PgStat_StatReplSlotEntry stats; } PgStatShared_ReplSlot; +typedef struct PgStatShared_Backend +{ + PgStatShared_Common header; + PgStat_Backend stats; +} PgStatShared_Backend; /* * Central shared memory entry for the cumulative stats system. @@ -604,6 +609,15 @@ extern void pgstat_archiver_init_shmem_cb(void *stats); extern void pgstat_archiver_reset_all_cb(TimestampTz ts); extern void pgstat_archiver_snapshot_cb(void); +/* + * Functions in pgstat_backend.c + */ + +extern void pgstat_flush_backend(bool nowait); + +extern PgStat_BackendPendingIO *pgstat_prep_backend_pending(ProcNumber procnum); +extern bool pgstat_backend_flush_cb(PgStat_EntryRef *entry_ref, bool nowait); +extern void pgstat_backend_reset_timestamp_cb(PgStatShared_Common *header, TimestampTz ts); /* * Functions in pgstat_bgwriter.c diff --git a/src/test/regress/expected/stats.out b/src/test/regress/expected/stats.out index 56771f83ed..3447e7b75d 100644 --- a/src/test/regress/expected/stats.out +++ b/src/test/regress/expected/stats.out @@ -1249,7 +1249,7 @@ SELECT pg_stat_get_subscription_stats(NULL); (1 row) --- Test that the following operations are tracked in pg_stat_io: +-- Test that the following operations are tracked in pg_stat_io and in backend stats: -- - reads of target blocks into shared buffers -- - writes of shared buffers to permanent storage -- - extends of relations using shared buffers @@ -1259,11 +1259,19 @@ SELECT pg_stat_get_subscription_stats(NULL); -- be sure of the state of shared buffers at the point the test is run. -- Create a regular table and insert some data to generate IOCONTEXT_NORMAL -- extends. +SELECT pid AS checkpointer_pid FROM pg_stat_activity + WHERE backend_type = 'checkpointer' \gset SELECT sum(extends) AS io_sum_shared_before_extends FROM pg_stat_io WHERE context = 'normal' AND object = 'relation' \gset +SELECT sum(extends) AS my_io_sum_shared_before_extends + FROM pg_stat_get_backend_io(pg_backend_pid()) + WHERE context = 'normal' AND object = 'relation' \gset SELECT sum(writes) AS writes, sum(fsyncs) AS fsyncs FROM pg_stat_io WHERE object = 'relation' \gset io_sum_shared_before_ +SELECT sum(writes) AS writes, sum(fsyncs) AS fsyncs + FROM pg_stat_get_backend_io(pg_backend_pid()) + WHERE object = 'relation' \gset my_io_sum_shared_before_ CREATE TABLE test_io_shared(a int); INSERT INTO test_io_shared SELECT i FROM generate_series(1,100)i; SELECT pg_stat_force_next_flush(); @@ -1280,8 +1288,17 @@ SELECT :io_sum_shared_after_extends > :io_sum_shared_before_extends; t (1 row) +SELECT sum(extends) AS my_io_sum_shared_after_extends + FROM pg_stat_get_backend_io(pg_backend_pid()) + WHERE context = 'normal' AND object = 'relation' \gset +SELECT :my_io_sum_shared_after_extends > :my_io_sum_shared_before_extends; + ?column? +---------- + t +(1 row) + -- After a checkpoint, there should be some additional IOCONTEXT_NORMAL writes --- and fsyncs. +-- and fsyncs in the global stats (not for the backend). -- See comment above for rationale for two explicit CHECKPOINTs. CHECKPOINT; CHECKPOINT; @@ -1301,6 +1318,31 @@ SELECT current_setting('fsync') = 'off' t (1 row) +SELECT sum(writes) AS writes, sum(fsyncs) AS fsyncs + FROM pg_stat_get_backend_io(pg_backend_pid()) + WHERE object = 'relation' \gset my_io_sum_shared_after_ +SELECT :my_io_sum_shared_after_writes >= :my_io_sum_shared_before_writes; + ?column? +---------- + t +(1 row) + +SELECT current_setting('fsync') = 'off' + OR (:my_io_sum_shared_after_fsyncs = :my_io_sum_shared_before_fsyncs + AND :my_io_sum_shared_after_fsyncs= 0); + ?column? +---------- + t +(1 row) + +-- Don't return any rows if querying other backend's stats that are excluded +-- from the backend stats collection (like the checkpointer). +SELECT count(1) = 0 FROM pg_stat_get_backend_io(:checkpointer_pid); + ?column? +---------- + t +(1 row) + -- Change the tablespace so that the table is rewritten directly, then SELECT -- from it to cause it to be read back into shared buffers. SELECT sum(reads) AS io_sum_shared_before_reads @@ -1521,6 +1563,8 @@ SELECT pg_stat_have_stats('io', 0, 0); SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS io_stats_pre_reset FROM pg_stat_io \gset +SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS my_io_stats_pre_reset + FROM pg_stat_get_backend_io(pg_backend_pid()) \gset SELECT pg_stat_reset_shared('io'); pg_stat_reset_shared ---------------------- @@ -1535,6 +1579,30 @@ SELECT :io_stats_post_reset < :io_stats_pre_reset; t (1 row) +SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS my_io_stats_post_reset + FROM pg_stat_get_backend_io(pg_backend_pid()) \gset +-- pg_stat_reset_shared() did not reset backend IO stats +SELECT :my_io_stats_pre_reset <= :my_io_stats_post_reset; + ?column? +---------- + t +(1 row) + +-- but pg_stat_reset_backend_stats() does +SELECT pg_stat_reset_backend_stats(pg_backend_pid()); + pg_stat_reset_backend_stats +----------------------------- + +(1 row) + +SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS my_io_stats_post_backend_reset + FROM pg_stat_get_backend_io(pg_backend_pid()) \gset +SELECT :my_io_stats_pre_reset > :my_io_stats_post_backend_reset; + ?column? +---------- + t +(1 row) + -- test BRIN index doesn't block HOT update CREATE TABLE brin_hot ( id integer PRIMARY KEY, diff --git a/src/test/regress/sql/stats.sql b/src/test/regress/sql/stats.sql index 7147cc2f89..9c925005be 100644 --- a/src/test/regress/sql/stats.sql +++ b/src/test/regress/sql/stats.sql @@ -595,7 +595,7 @@ SELECT pg_stat_get_replication_slot(NULL); SELECT pg_stat_get_subscription_stats(NULL); --- Test that the following operations are tracked in pg_stat_io: +-- Test that the following operations are tracked in pg_stat_io and in backend stats: -- - reads of target blocks into shared buffers -- - writes of shared buffers to permanent storage -- - extends of relations using shared buffers @@ -607,20 +607,32 @@ SELECT pg_stat_get_subscription_stats(NULL); -- Create a regular table and insert some data to generate IOCONTEXT_NORMAL -- extends. +SELECT pid AS checkpointer_pid FROM pg_stat_activity + WHERE backend_type = 'checkpointer' \gset SELECT sum(extends) AS io_sum_shared_before_extends FROM pg_stat_io WHERE context = 'normal' AND object = 'relation' \gset +SELECT sum(extends) AS my_io_sum_shared_before_extends + FROM pg_stat_get_backend_io(pg_backend_pid()) + WHERE context = 'normal' AND object = 'relation' \gset SELECT sum(writes) AS writes, sum(fsyncs) AS fsyncs FROM pg_stat_io WHERE object = 'relation' \gset io_sum_shared_before_ +SELECT sum(writes) AS writes, sum(fsyncs) AS fsyncs + FROM pg_stat_get_backend_io(pg_backend_pid()) + WHERE object = 'relation' \gset my_io_sum_shared_before_ CREATE TABLE test_io_shared(a int); INSERT INTO test_io_shared SELECT i FROM generate_series(1,100)i; SELECT pg_stat_force_next_flush(); SELECT sum(extends) AS io_sum_shared_after_extends FROM pg_stat_io WHERE context = 'normal' AND object = 'relation' \gset SELECT :io_sum_shared_after_extends > :io_sum_shared_before_extends; +SELECT sum(extends) AS my_io_sum_shared_after_extends + FROM pg_stat_get_backend_io(pg_backend_pid()) + WHERE context = 'normal' AND object = 'relation' \gset +SELECT :my_io_sum_shared_after_extends > :my_io_sum_shared_before_extends; -- After a checkpoint, there should be some additional IOCONTEXT_NORMAL writes --- and fsyncs. +-- and fsyncs in the global stats (not for the backend). -- See comment above for rationale for two explicit CHECKPOINTs. CHECKPOINT; CHECKPOINT; @@ -630,6 +642,17 @@ SELECT sum(writes) AS writes, sum(fsyncs) AS fsyncs SELECT :io_sum_shared_after_writes > :io_sum_shared_before_writes; SELECT current_setting('fsync') = 'off' OR :io_sum_shared_after_fsyncs > :io_sum_shared_before_fsyncs; +SELECT sum(writes) AS writes, sum(fsyncs) AS fsyncs + FROM pg_stat_get_backend_io(pg_backend_pid()) + WHERE object = 'relation' \gset my_io_sum_shared_after_ +SELECT :my_io_sum_shared_after_writes >= :my_io_sum_shared_before_writes; +SELECT current_setting('fsync') = 'off' + OR (:my_io_sum_shared_after_fsyncs = :my_io_sum_shared_before_fsyncs + AND :my_io_sum_shared_after_fsyncs= 0); + +-- Don't return any rows if querying other backend's stats that are excluded +-- from the backend stats collection (like the checkpointer). +SELECT count(1) = 0 FROM pg_stat_get_backend_io(:checkpointer_pid); -- Change the tablespace so that the table is rewritten directly, then SELECT -- from it to cause it to be read back into shared buffers. @@ -762,10 +785,21 @@ SELECT :io_sum_bulkwrite_strategy_extends_after > :io_sum_bulkwrite_strategy_ext SELECT pg_stat_have_stats('io', 0, 0); SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS io_stats_pre_reset FROM pg_stat_io \gset +SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS my_io_stats_pre_reset + FROM pg_stat_get_backend_io(pg_backend_pid()) \gset SELECT pg_stat_reset_shared('io'); SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS io_stats_post_reset FROM pg_stat_io \gset SELECT :io_stats_post_reset < :io_stats_pre_reset; +SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS my_io_stats_post_reset + FROM pg_stat_get_backend_io(pg_backend_pid()) \gset +-- pg_stat_reset_shared() did not reset backend IO stats +SELECT :my_io_stats_pre_reset <= :my_io_stats_post_reset; +-- but pg_stat_reset_backend_stats() does +SELECT pg_stat_reset_backend_stats(pg_backend_pid()); +SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS my_io_stats_post_backend_reset + FROM pg_stat_get_backend_io(pg_backend_pid()) \gset +SELECT :my_io_stats_pre_reset > :my_io_stats_post_backend_reset; -- test BRIN index doesn't block HOT update diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index ce33e55bf1..398dd92527 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2121,6 +2121,7 @@ PgFdwSamplingMethod PgFdwScanState PgIfAddrCallback PgStatShared_Archiver +PgStatShared_Backend PgStatShared_BgWriter PgStatShared_Checkpointer PgStatShared_Common @@ -2136,6 +2137,8 @@ PgStatShared_SLRU PgStatShared_Subscription PgStatShared_Wal PgStat_ArchiverStats +PgStat_Backend +PgStat_BackendPendingIO PgStat_BackendSubEntry PgStat_BgWriterStats PgStat_BktypeIO -- 2.34.1 --Sc8S/0qAVNqKo8ol-- ^ permalink raw reply [nested|flat] 25+ messages in thread
* [PATCH v5 1/4] per backend I/O statistics @ 2024-10-28 12:50 Bertrand Drouvot <[email protected]> 0 siblings, 0 replies; 25+ messages in thread From: Bertrand Drouvot @ 2024-10-28 12:50 UTC (permalink / raw) While pg_stat_io provides cluster-wide I/O statistics, this commit adds a new statistics kind and a new pg_my_stat_io view to display "my" backend I/O statistics. The new KIND is named PGSTAT_KIND_PER_BACKEND as it could be used in the future to store other statistics (than the I/O ones) per backend. The new KIND is a variable-numbered one and has an automatic cap on the maximum number of entries (as its hash key contains the proc number). There is no need to serialize the per backend I/O stats to disk (no point to see stats for backends that do not exist anymore after a re-start), so a new "to_serialize" field is added in the PgStat_KindInfo struct. Also adding a new function pg_stat_reset_single_backend_io_counters() to be able to reset the I/O stats for a given backend pid. A subsequent commit will add a new pg_stat_get_backend_io() function to be able to retrieve the I/O statistics for a given backend pid. XXX: Bump catalog version needs to be done. --- doc/src/sgml/config.sgml | 4 +- doc/src/sgml/monitoring.sgml | 43 ++++++ src/backend/catalog/system_functions.sql | 2 + src/backend/catalog/system_views.sql | 22 ++++ src/backend/utils/activity/backend_status.c | 3 + src/backend/utils/activity/pgstat.c | 34 ++++- src/backend/utils/activity/pgstat_io.c | 114 ++++++++++++++-- src/backend/utils/activity/pgstat_shmem.c | 30 ++++- src/backend/utils/adt/pgstatfuncs.c | 124 ++++++++++++++++++ src/include/catalog/pg_proc.dat | 14 ++ src/include/pgstat.h | 27 +++- src/include/utils/pgstat_internal.h | 13 ++ .../injection_points/injection_stats.c | 1 + src/test/regress/expected/rules.out | 19 +++ src/test/regress/expected/stats.out | 60 ++++++++- src/test/regress/sql/stats.sql | 31 ++++- src/tools/pgindent/typedefs.list | 2 + 17 files changed, 517 insertions(+), 26 deletions(-) 10.2% doc/src/sgml/ 29.7% src/backend/utils/activity/ 19.2% src/backend/utils/adt/ 5.1% src/include/catalog/ 7.2% src/include/ 14.7% src/test/regress/expected/ 10.4% src/test/regress/sql/ 3.1% src/ diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index d54f904956..fbabf371a6 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -8361,7 +8361,9 @@ COPY postgres_log FROM '/full/path/to/logfile.csv' WITH csv; displayed in <link linkend="monitoring-pg-stat-database-view"> <structname>pg_stat_database</structname></link>, <link linkend="monitoring-pg-stat-io-view"> - <structname>pg_stat_io</structname></link>, in the output of + <structname>pg_stat_io</structname></link>, + <link linkend="monitoring-pg-my-stat-io-view"> + <structname>pg_my_stat_io</structname></link>, in the output of <xref linkend="sql-explain"/> when the <literal>BUFFERS</literal> option is used, in the output of <xref linkend="sql-vacuum"/> when the <literal>VERBOSE</literal> option is used, by autovacuum diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml index 331315f8d3..fc6aded3da 100644 --- a/doc/src/sgml/monitoring.sgml +++ b/doc/src/sgml/monitoring.sgml @@ -488,6 +488,16 @@ postgres 27093 0.0 0.0 30096 2752 ? Ss 11:34 0:00 postgres: ser </entry> </row> + <row> + <entry><structname>pg_my_stat_io</structname><indexterm><primary>pg_my_stat_io</primary></indexterm></entry> + <entry> + One row for each combination of context and target object containing + my backend I/O statistics. + See <link linkend="monitoring-pg-my-stat-io-view"> + <structname>pg_my_stat_io</structname></link> for details. + </entry> + </row> + <row> <entry><structname>pg_stat_replication_slots</structname><indexterm><primary>pg_stat_replication_slots</primary></indexterm></entry> <entry>One row per replication slot, showing statistics about the @@ -2946,7 +2956,23 @@ description | Waiting for a newly initialized WAL file to reach durable storage </para> </note> + </sect2> + <sect2 id="monitoring-pg-my-stat-io-view"> + <title><structname>pg_my_stat_io</structname></title> + + <indexterm> + <primary>pg_my_stat_io</primary> + </indexterm> + + <para> + The <structname>pg_my_stat_io</structname> view will contain one row for each + combination of target I/O object and I/O context, showing + my backend I/O statistics. Combinations which do not make sense are + omitted. The fields are exactly the same as the ones in the <link + linkend="monitoring-pg-stat-io-view"> <structname>pg_stat_io</structname></link> + view. + </para> </sect2> @@ -4971,6 +4997,23 @@ description | Waiting for a newly initialized WAL file to reach durable storage </para></entry> </row> + <row> + <entry role="func_table_entry"><para role="func_signature"> + <indexterm> + <primary>pg_stat_reset_single_backend_io_counters</primary> + </indexterm> + <function>pg_stat_reset_single_backend_io_counters</function> ( <type>int4</type> ) + <returnvalue>void</returnvalue> + </para> + <para> + Resets I/O statistics for a single backend to zero. + </para> + <para> + This function is restricted to superusers by default, but other users + can be granted EXECUTE to run the function. + </para></entry> + </row> + <row> <entry role="func_table_entry"><para role="func_signature"> <indexterm> diff --git a/src/backend/catalog/system_functions.sql b/src/backend/catalog/system_functions.sql index c51dfca802..8a1ae44fca 100644 --- a/src/backend/catalog/system_functions.sql +++ b/src/backend/catalog/system_functions.sql @@ -711,6 +711,8 @@ REVOKE EXECUTE ON FUNCTION pg_stat_reset_single_table_counters(oid) FROM public; REVOKE EXECUTE ON FUNCTION pg_stat_reset_single_function_counters(oid) FROM public; +REVOKE EXECUTE ON FUNCTION pg_stat_reset_single_backend_io_counters(int4) FROM public; + REVOKE EXECUTE ON FUNCTION pg_stat_reset_replication_slot(text) FROM public; REVOKE EXECUTE ON FUNCTION pg_stat_have_stats(text, oid, int8) FROM public; diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index 3456b821bc..09af4a40a8 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1170,6 +1170,28 @@ SELECT b.stats_reset FROM pg_stat_get_io() b; +CREATE VIEW pg_my_stat_io AS +SELECT + b.backend_type, + b.object, + b.context, + b.reads, + b.read_time, + b.writes, + b.write_time, + b.writebacks, + b.writeback_time, + b.extends, + b.extend_time, + b.op_bytes, + b.hits, + b.evictions, + b.reuses, + b.fsyncs, + b.fsync_time, + b.stats_reset +FROM pg_stat_get_my_io() b; + CREATE VIEW pg_stat_wal AS SELECT w.wal_records, diff --git a/src/backend/utils/activity/backend_status.c b/src/backend/utils/activity/backend_status.c index bdb3a296ca..aa683e150c 100644 --- a/src/backend/utils/activity/backend_status.c +++ b/src/backend/utils/activity/backend_status.c @@ -249,6 +249,9 @@ pgstat_beinit(void) Assert(MyProcNumber >= 0 && MyProcNumber < NumBackendStatSlots); MyBEEntry = &BackendStatusArray[MyProcNumber]; + /* Create the per backend stat entry */ + pgstat_create_backend_stat(MyProcNumber); + /* Set up a process-exit hook to clean up */ on_shmem_exit(pgstat_beshutdown_hook, 0); } diff --git a/src/backend/utils/activity/pgstat.c b/src/backend/utils/activity/pgstat.c index ea8c5691e8..aacf61c9a4 100644 --- a/src/backend/utils/activity/pgstat.c +++ b/src/backend/utils/activity/pgstat.c @@ -281,6 +281,7 @@ static const PgStat_KindInfo pgstat_kind_builtin_infos[PGSTAT_KIND_BUILTIN_SIZE] .name = "database", .fixed_amount = false, + .to_serialize = true, /* so pg_stat_database entries can be seen in all databases */ .accessed_across_databases = true, @@ -297,6 +298,7 @@ static const PgStat_KindInfo pgstat_kind_builtin_infos[PGSTAT_KIND_BUILTIN_SIZE] .name = "relation", .fixed_amount = false, + .to_serialize = true, .shared_size = sizeof(PgStatShared_Relation), .shared_data_off = offsetof(PgStatShared_Relation, stats), @@ -311,6 +313,7 @@ static const PgStat_KindInfo pgstat_kind_builtin_infos[PGSTAT_KIND_BUILTIN_SIZE] .name = "function", .fixed_amount = false, + .to_serialize = true, .shared_size = sizeof(PgStatShared_Function), .shared_data_off = offsetof(PgStatShared_Function, stats), @@ -324,6 +327,7 @@ static const PgStat_KindInfo pgstat_kind_builtin_infos[PGSTAT_KIND_BUILTIN_SIZE] .name = "replslot", .fixed_amount = false, + .to_serialize = true, .accessed_across_databases = true, @@ -340,6 +344,7 @@ static const PgStat_KindInfo pgstat_kind_builtin_infos[PGSTAT_KIND_BUILTIN_SIZE] .name = "subscription", .fixed_amount = false, + .to_serialize = true, /* so pg_stat_subscription_stats entries can be seen in all databases */ .accessed_across_databases = true, @@ -352,6 +357,22 @@ static const PgStat_KindInfo pgstat_kind_builtin_infos[PGSTAT_KIND_BUILTIN_SIZE] .reset_timestamp_cb = pgstat_subscription_reset_timestamp_cb, }, + [PGSTAT_KIND_PER_BACKEND] = { + .name = "perbackend", + + .fixed_amount = false, + .to_serialize = false, + + .accessed_across_databases = true, + + .shared_size = sizeof(PgStatShared_Backend), + .shared_data_off = offsetof(PgStatShared_Backend, stats), + .shared_data_len = sizeof(((PgStatShared_Backend *) 0)->stats), + .pending_size = sizeof(PgStat_PendingIO), + + .flush_pending_cb = pgstat_per_backend_flush_cb, + .reset_timestamp_cb = pgstat_backend_reset_timestamp_cb, + }, /* stats for fixed-numbered (mostly 1) objects */ @@ -359,6 +380,7 @@ static const PgStat_KindInfo pgstat_kind_builtin_infos[PGSTAT_KIND_BUILTIN_SIZE] .name = "archiver", .fixed_amount = true, + .to_serialize = true, .snapshot_ctl_off = offsetof(PgStat_Snapshot, archiver), .shared_ctl_off = offsetof(PgStat_ShmemControl, archiver), @@ -374,6 +396,7 @@ static const PgStat_KindInfo pgstat_kind_builtin_infos[PGSTAT_KIND_BUILTIN_SIZE] .name = "bgwriter", .fixed_amount = true, + .to_serialize = true, .snapshot_ctl_off = offsetof(PgStat_Snapshot, bgwriter), .shared_ctl_off = offsetof(PgStat_ShmemControl, bgwriter), @@ -389,6 +412,7 @@ static const PgStat_KindInfo pgstat_kind_builtin_infos[PGSTAT_KIND_BUILTIN_SIZE] .name = "checkpointer", .fixed_amount = true, + .to_serialize = true, .snapshot_ctl_off = offsetof(PgStat_Snapshot, checkpointer), .shared_ctl_off = offsetof(PgStat_ShmemControl, checkpointer), @@ -404,6 +428,7 @@ static const PgStat_KindInfo pgstat_kind_builtin_infos[PGSTAT_KIND_BUILTIN_SIZE] .name = "io", .fixed_amount = true, + .to_serialize = true, .snapshot_ctl_off = offsetof(PgStat_Snapshot, io), .shared_ctl_off = offsetof(PgStat_ShmemControl, io), @@ -421,6 +446,7 @@ static const PgStat_KindInfo pgstat_kind_builtin_infos[PGSTAT_KIND_BUILTIN_SIZE] .name = "slru", .fixed_amount = true, + .to_serialize = true, .snapshot_ctl_off = offsetof(PgStat_Snapshot, slru), .shared_ctl_off = offsetof(PgStat_ShmemControl, slru), @@ -438,6 +464,7 @@ static const PgStat_KindInfo pgstat_kind_builtin_infos[PGSTAT_KIND_BUILTIN_SIZE] .name = "wal", .fixed_amount = true, + .to_serialize = true, .snapshot_ctl_off = offsetof(PgStat_Snapshot, wal), .shared_ctl_off = offsetof(PgStat_ShmemControl, wal), @@ -515,7 +542,7 @@ pgstat_discard_stats(void) /* * Reset stats contents. This will set reset timestamps of fixed-numbered - * stats to the current time (no variable stats exist). + * stats to the current time (the per-backend variable stat exists too). */ pgstat_reset_after_failure(); } @@ -756,7 +783,7 @@ pgstat_report_stat(bool force) partial_flush = false; - /* flush database / relation / function / ... stats */ + /* flush database / relation / function / backend / ... stats */ partial_flush |= pgstat_flush_pending_entries(nowait); /* flush of fixed-numbered stats */ @@ -1660,6 +1687,9 @@ pgstat_write_statsfile(XLogRecPtr redo) kind_info = pgstat_get_kind_info(ps->key.kind); + if (!kind_info->to_serialize) + continue; + /* if not dropped the valid-entry refcount should exist */ Assert(pg_atomic_read_u32(&ps->refcount) > 0); diff --git a/src/backend/utils/activity/pgstat_io.c b/src/backend/utils/activity/pgstat_io.c index cc2ffc78aa..a2bb11adc9 100644 --- a/src/backend/utils/activity/pgstat_io.c +++ b/src/backend/utils/activity/pgstat_io.c @@ -21,13 +21,6 @@ #include "utils/pgstat_internal.h" -typedef struct PgStat_PendingIO -{ - PgStat_Counter counts[IOOBJECT_NUM_TYPES][IOCONTEXT_NUM_TYPES][IOOP_NUM_TYPES]; - instr_time pending_times[IOOBJECT_NUM_TYPES][IOCONTEXT_NUM_TYPES][IOOP_NUM_TYPES]; -} PgStat_PendingIO; - - static PgStat_PendingIO PendingIOStats; static bool have_iostats = false; @@ -82,12 +75,17 @@ pgstat_count_io_op(IOObject io_object, IOContext io_context, IOOp io_op) void pgstat_count_io_op_n(IOObject io_object, IOContext io_context, IOOp io_op, uint32 cnt) { + PgStat_PendingIO *entry_ref; + Assert((unsigned int) io_object < IOOBJECT_NUM_TYPES); Assert((unsigned int) io_context < IOCONTEXT_NUM_TYPES); Assert((unsigned int) io_op < IOOP_NUM_TYPES); Assert(pgstat_tracks_io_op(MyBackendType, io_object, io_context, io_op)); + entry_ref = pgstat_prep_per_backend_pending(MyProcNumber); + PendingIOStats.counts[io_object][io_context][io_op] += cnt; + entry_ref->counts[io_object][io_context][io_op] += cnt; have_iostats = true; } @@ -122,6 +120,10 @@ void pgstat_count_io_op_time(IOObject io_object, IOContext io_context, IOOp io_op, instr_time start_time, uint32 cnt) { + PgStat_PendingIO *entry_ref; + + entry_ref = pgstat_prep_per_backend_pending(MyProcNumber); + if (track_io_timing) { instr_time io_time; @@ -148,6 +150,8 @@ pgstat_count_io_op_time(IOObject io_object, IOContext io_context, IOOp io_op, INSTR_TIME_ADD(PendingIOStats.pending_times[io_object][io_context][io_op], io_time); + INSTR_TIME_ADD(entry_ref->pending_times[io_object][io_context][io_op], + io_time); } pgstat_count_io_op_n(io_object, io_context, io_op, cnt); @@ -161,6 +165,13 @@ pgstat_fetch_stat_io(void) return &pgStatLocal.snapshot.io; } +PgStat_Backend * +pgstat_fetch_my_stat_io(void) +{ + return (PgStat_Backend *) + pgstat_fetch_entry(PGSTAT_KIND_PER_BACKEND, InvalidOid, MyProcNumber); +} + /* * Check if there any IO stats waiting for flush. */ @@ -171,12 +182,18 @@ pgstat_io_have_pending_cb(void) } /* - * Simpler wrapper of pgstat_io_flush_cb() + * Simpler wrapper of pgstat_io_flush_cb() and pgstat_per_backend_flush_cb(). */ void pgstat_flush_io(bool nowait) { + PgStat_EntryRef *entry_ref; + + entry_ref = pgstat_get_entry_ref(PGSTAT_KIND_PER_BACKEND, InvalidOid, + MyProcNumber, false, NULL); + (void) pgstat_io_flush_cb(nowait); + (void) pgstat_per_backend_flush_cb(entry_ref, nowait); } /* @@ -235,6 +252,49 @@ pgstat_io_flush_cb(bool nowait) return false; } +/* + * Flush out locally pending backend statistics + * + * If no stats have been recorded, this function returns false. + */ +bool +pgstat_per_backend_flush_cb(PgStat_EntryRef *entry_ref, bool nowait) +{ + PgStatShared_Backend *shbackendioent; + PgStat_PendingIO *pendingent; + PgStat_BktypeIO *bktype_shstats; + + if (!pgstat_lock_entry(entry_ref, nowait)) + return false; + + shbackendioent = (PgStatShared_Backend *) entry_ref->shared_stats; + bktype_shstats = &shbackendioent->stats.stats; + pendingent = (PgStat_PendingIO *) entry_ref->pending; + + for (int io_object = 0; io_object < IOOBJECT_NUM_TYPES; io_object++) + { + for (int io_context = 0; io_context < IOCONTEXT_NUM_TYPES; io_context++) + { + for (int io_op = 0; io_op < IOOP_NUM_TYPES; io_op++) + { + instr_time time; + + bktype_shstats->counts[io_object][io_context][io_op] += + pendingent->counts[io_object][io_context][io_op]; + + time = pendingent->pending_times[io_object][io_context][io_op]; + + bktype_shstats->times[io_object][io_context][io_op] += + INSTR_TIME_GET_MICROSEC(time); + } + } + } + + pgstat_unlock_entry(entry_ref); + + return true; +} + const char * pgstat_get_io_context_name(IOContext io_context) { @@ -325,6 +385,38 @@ pgstat_io_snapshot_cb(void) } } +void +pgstat_create_backend_stat(ProcNumber procnum) +{ + PgStat_EntryRef *entry_ref; + PgStatShared_Backend *shstatent; + + entry_ref = pgstat_prep_pending_entry(PGSTAT_KIND_PER_BACKEND, InvalidOid, + procnum, NULL); + + shstatent = (PgStatShared_Backend *) entry_ref->shared_stats; + + /* + * NB: need to accept that there might be stats from an older backend, + * e.g. if we previously used this proc number. + */ + memset(&shstatent->stats, 0, sizeof(shstatent->stats)); + + /* set the backend type */ + shstatent->stats.bktype = MyBackendType; +} + +PgStat_PendingIO * +pgstat_prep_per_backend_pending(ProcNumber procnum) +{ + PgStat_EntryRef *entry_ref; + + entry_ref = pgstat_prep_pending_entry(PGSTAT_KIND_PER_BACKEND, InvalidOid, + procnum, NULL); + + return entry_ref->pending; +} + /* * IO statistics are not collected for all BackendTypes. * @@ -503,3 +595,9 @@ pgstat_tracks_io_op(BackendType bktype, IOObject io_object, return true; } + +void +pgstat_backend_reset_timestamp_cb(PgStatShared_Common *header, TimestampTz ts) +{ + ((PgStatShared_Backend *) header)->stats.stat_reset_timestamp = ts; +} diff --git a/src/backend/utils/activity/pgstat_shmem.c b/src/backend/utils/activity/pgstat_shmem.c index c1b7ff76b1..b871c1d7d0 100644 --- a/src/backend/utils/activity/pgstat_shmem.c +++ b/src/backend/utils/activity/pgstat_shmem.c @@ -813,13 +813,24 @@ pgstat_free_entry(PgStatShared_HashEntry *shent, dshash_seq_status *hstat) */ static bool pgstat_drop_entry_internal(PgStatShared_HashEntry *shent, - dshash_seq_status *hstat) + dshash_seq_status *hstat, bool ignore_aux_proc) { Assert(shent->body != InvalidDsaPointer); /* should already have released local reference */ if (pgStatEntryRefHash) - Assert(!pgstat_entry_ref_hash_lookup(pgStatEntryRefHash, shent->key)); + { + /* + * The following assertion is not correct when coming from + * pgstat_discard_stats() and for per backend stats linked to an + * auxiliary process. + */ + if (ignore_aux_proc && shent->key.kind == PGSTAT_KIND_PER_BACKEND && + MyProcNumber >= MaxBackends) + return false; + else + Assert(!pgstat_entry_ref_hash_lookup(pgStatEntryRefHash, shent->key)); + } /* * Signal that the entry is dropped - this will eventually cause other @@ -882,7 +893,7 @@ pgstat_drop_database_and_contents(Oid dboid) if (p->key.dboid != dboid) continue; - if (!pgstat_drop_entry_internal(p, &hstat)) + if (!pgstat_drop_entry_internal(p, &hstat, false)) { /* * Even statistics for a dropped database might currently be @@ -941,7 +952,7 @@ pgstat_drop_entry(PgStat_Kind kind, Oid dboid, uint64 objid) shent = dshash_find(pgStatLocal.shared_hash, &key, true); if (shent) { - freed = pgstat_drop_entry_internal(shent, NULL); + freed = pgstat_drop_entry_internal(shent, NULL, false); /* * Database stats contain other stats. Drop those as well when @@ -969,7 +980,7 @@ pgstat_drop_all_entries(void) if (ps->dropped) continue; - if (!pgstat_drop_entry_internal(ps, &hstat)) + if (!pgstat_drop_entry_internal(ps, &hstat, true)) not_freed_count++; } dshash_seq_term(&hstat); @@ -982,11 +993,20 @@ static void shared_stat_reset_contents(PgStat_Kind kind, PgStatShared_Common *header, TimestampTz ts) { + BackendType bktype; const PgStat_KindInfo *kind_info = pgstat_get_kind_info(kind); + /* save the bktype */ + if (kind == PGSTAT_KIND_PER_BACKEND) + bktype = ((PgStatShared_Backend *) header)->stats.bktype; + memset(pgstat_get_entry_data(kind, header), 0, pgstat_get_entry_len(kind)); + /* restore the bktype */ + if (kind == PGSTAT_KIND_PER_BACKEND) + ((PgStatShared_Backend *) header)->stats.bktype = bktype; + if (kind_info->reset_timestamp_cb) kind_info->reset_timestamp_cb(header, ts); } diff --git a/src/backend/utils/adt/pgstatfuncs.c b/src/backend/utils/adt/pgstatfuncs.c index f7b50e0b5a..fd16986156 100644 --- a/src/backend/utils/adt/pgstatfuncs.c +++ b/src/backend/utils/adt/pgstatfuncs.c @@ -1474,6 +1474,111 @@ pg_stat_get_io(PG_FUNCTION_ARGS) return (Datum) 0; } +Datum +pg_stat_get_my_io(PG_FUNCTION_ARGS) +{ + ReturnSetInfo *rsinfo; + PgStat_Backend *backend_stats; + Datum bktype_desc; + PgStat_BktypeIO *bktype_stats; + BackendType bktype; + Datum reset_time; + + InitMaterializedSRF(fcinfo, 0); + rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; + + backend_stats = pgstat_fetch_my_stat_io(); + + bktype = backend_stats->bktype; + bktype_desc = CStringGetTextDatum(GetBackendTypeDesc(bktype)); + bktype_stats = &backend_stats->stats; + reset_time = TimestampTzGetDatum(backend_stats->stat_reset_timestamp); + + /* + * In Assert builds, we can afford an extra loop through all of the + * counters checking that only expected stats are non-zero, since it keeps + * the non-Assert code cleaner. + */ + Assert(pgstat_bktype_io_stats_valid(bktype_stats, bktype)); + + for (int io_obj = 0; io_obj < IOOBJECT_NUM_TYPES; io_obj++) + { + const char *obj_name = pgstat_get_io_object_name(io_obj); + + for (int io_context = 0; io_context < IOCONTEXT_NUM_TYPES; io_context++) + { + const char *context_name = pgstat_get_io_context_name(io_context); + + Datum values[IO_NUM_COLUMNS] = {0}; + bool nulls[IO_NUM_COLUMNS] = {0}; + + /* + * Some combinations of BackendType, IOObject, and IOContext are + * not valid for any type of IOOp. In such cases, omit the entire + * row from the view. + */ + if (!pgstat_tracks_io_object(bktype, io_obj, io_context)) + continue; + + values[IO_COL_BACKEND_TYPE] = bktype_desc; + values[IO_COL_CONTEXT] = CStringGetTextDatum(context_name); + values[IO_COL_OBJECT] = CStringGetTextDatum(obj_name); + if (backend_stats->stat_reset_timestamp != 0) + values[IO_COL_RESET_TIME] = reset_time; + else + nulls[IO_COL_RESET_TIME] = true; + + /* + * Hard-code this to the value of BLCKSZ for now. Future values + * could include XLOG_BLCKSZ, once WAL IO is tracked, and constant + * multipliers, once non-block-oriented IO (e.g. temporary file + * IO) is tracked. + */ + values[IO_COL_CONVERSION] = Int64GetDatum(BLCKSZ); + + for (int io_op = 0; io_op < IOOP_NUM_TYPES; io_op++) + { + int op_idx = pgstat_get_io_op_index(io_op); + int time_idx = pgstat_get_io_time_index(io_op); + + /* + * Some combinations of BackendType and IOOp, of IOContext and + * IOOp, and of IOObject and IOOp are not tracked. Set these + * cells in the view NULL. + */ + if (pgstat_tracks_io_op(bktype, io_obj, io_context, io_op)) + { + PgStat_Counter count = + bktype_stats->counts[io_obj][io_context][io_op]; + + values[op_idx] = Int64GetDatum(count); + } + else + nulls[op_idx] = true; + + /* not every operation is timed */ + if (time_idx == IO_COL_INVALID) + continue; + + if (!nulls[op_idx]) + { + PgStat_Counter time = + bktype_stats->times[io_obj][io_context][io_op]; + + values[time_idx] = Float8GetDatum(pg_stat_us_to_ms(time)); + } + else + nulls[time_idx] = true; + } + + tuplestore_putvalues(rsinfo->setResult, rsinfo->setDesc, + values, nulls); + } + } + + return (Datum) 0; +} + /* * Returns statistics of WAL activity */ @@ -1722,6 +1827,7 @@ pg_stat_reset_shared(PG_FUNCTION_ARGS) pgstat_reset_of_kind(PGSTAT_KIND_BGWRITER); pgstat_reset_of_kind(PGSTAT_KIND_CHECKPOINTER); pgstat_reset_of_kind(PGSTAT_KIND_IO); + pgstat_reset_of_kind(PGSTAT_KIND_PER_BACKEND); XLogPrefetchResetStats(); pgstat_reset_of_kind(PGSTAT_KIND_SLRU); pgstat_reset_of_kind(PGSTAT_KIND_WAL); @@ -1779,6 +1885,24 @@ pg_stat_reset_single_function_counters(PG_FUNCTION_ARGS) PG_RETURN_VOID(); } +Datum +pg_stat_reset_single_backend_io_counters(PG_FUNCTION_ARGS) +{ + PGPROC *proc; + int backend_pid = PG_GETARG_INT32(0); + + proc = BackendPidGetProc(backend_pid); + + /* Maybe an auxiliary process? */ + if (proc == NULL) + proc = AuxiliaryPidGetProc(backend_pid); + + if (proc) + pgstat_reset(PGSTAT_KIND_PER_BACKEND, InvalidOid, GetNumberFromPGProc(proc)); + + PG_RETURN_VOID(); +} + /* Reset SLRU counters (a specific one or all of them). */ Datum pg_stat_reset_slru(PG_FUNCTION_ARGS) diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index f23321a41f..89eb89efe4 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -5903,6 +5903,15 @@ proargnames => '{backend_type,object,context,reads,read_time,writes,write_time,writebacks,writeback_time,extends,extend_time,op_bytes,hits,evictions,reuses,fsyncs,fsync_time,stats_reset}', prosrc => 'pg_stat_get_io' }, +{ oid => '8806', descr => 'statistics: my backend IO statistics', + proname => 'pg_stat_get_my_io', prorows => '5', proretset => 't', + provolatile => 'v', proparallel => 'r', prorettype => 'record', + proargtypes => '', + proallargtypes => '{text,text,text,int8,float8,int8,float8,int8,float8,int8,float8,int8,int8,int8,int8,int8,float8,timestamptz}', + proargmodes => '{o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o}', + proargnames => '{backend_type,object,context,reads,read_time,writes,write_time,writebacks,writeback_time,extends,extend_time,op_bytes,hits,evictions,reuses,fsyncs,fsync_time,stats_reset}', + prosrc => 'pg_stat_get_my_io' }, + { oid => '1136', descr => 'statistics: information about WAL activity', proname => 'pg_stat_get_wal', proisstrict => 'f', provolatile => 's', proparallel => 'r', prorettype => 'record', proargtypes => '', @@ -6042,6 +6051,11 @@ proname => 'pg_stat_reset_single_function_counters', provolatile => 'v', prorettype => 'void', proargtypes => 'oid', prosrc => 'pg_stat_reset_single_function_counters' }, +{ oid => '9987', + descr => 'statistics: reset collected IO statistics for a single backend', + proname => 'pg_stat_reset_single_backend_io_counters', provolatile => 'v', + prorettype => 'void', proargtypes => 'int4', + prosrc => 'pg_stat_reset_single_backend_io_counters' }, { oid => '2307', descr => 'statistics: reset collected statistics for a single SLRU', proname => 'pg_stat_reset_slru', proisstrict => 'f', provolatile => 'v', diff --git a/src/include/pgstat.h b/src/include/pgstat.h index df53fa2d4f..13b9c71759 100644 --- a/src/include/pgstat.h +++ b/src/include/pgstat.h @@ -49,14 +49,15 @@ #define PGSTAT_KIND_FUNCTION 3 /* per-function statistics */ #define PGSTAT_KIND_REPLSLOT 4 /* per-slot statistics */ #define PGSTAT_KIND_SUBSCRIPTION 5 /* per-subscription statistics */ +#define PGSTAT_KIND_PER_BACKEND 6 /* stats for fixed-numbered objects */ -#define PGSTAT_KIND_ARCHIVER 6 -#define PGSTAT_KIND_BGWRITER 7 -#define PGSTAT_KIND_CHECKPOINTER 8 -#define PGSTAT_KIND_IO 9 -#define PGSTAT_KIND_SLRU 10 -#define PGSTAT_KIND_WAL 11 +#define PGSTAT_KIND_ARCHIVER 7 +#define PGSTAT_KIND_BGWRITER 8 +#define PGSTAT_KIND_CHECKPOINTER 9 +#define PGSTAT_KIND_IO 10 +#define PGSTAT_KIND_SLRU 11 +#define PGSTAT_KIND_WAL 12 #define PGSTAT_KIND_BUILTIN_MIN PGSTAT_KIND_DATABASE #define PGSTAT_KIND_BUILTIN_MAX PGSTAT_KIND_WAL @@ -347,12 +348,24 @@ typedef struct PgStat_BktypeIO PgStat_Counter times[IOOBJECT_NUM_TYPES][IOCONTEXT_NUM_TYPES][IOOP_NUM_TYPES]; } PgStat_BktypeIO; +typedef struct PgStat_PendingIO +{ + PgStat_Counter counts[IOOBJECT_NUM_TYPES][IOCONTEXT_NUM_TYPES][IOOP_NUM_TYPES]; + instr_time pending_times[IOOBJECT_NUM_TYPES][IOCONTEXT_NUM_TYPES][IOOP_NUM_TYPES]; +} PgStat_PendingIO; + typedef struct PgStat_IO { TimestampTz stat_reset_timestamp; PgStat_BktypeIO stats[BACKEND_NUM_TYPES]; } PgStat_IO; +typedef struct PgStat_Backend +{ + TimestampTz stat_reset_timestamp; + BackendType bktype; + PgStat_BktypeIO stats; +} PgStat_Backend; typedef struct PgStat_StatDBEntry { @@ -562,6 +575,7 @@ extern void pgstat_count_io_op_time(IOObject io_object, IOContext io_context, IOOp io_op, instr_time start_time, uint32 cnt); extern PgStat_IO *pgstat_fetch_stat_io(void); +extern PgStat_Backend *pgstat_fetch_my_stat_io(void); extern const char *pgstat_get_io_context_name(IOContext io_context); extern const char *pgstat_get_io_object_name(IOObject io_object); @@ -570,6 +584,7 @@ extern bool pgstat_tracks_io_object(BackendType bktype, IOObject io_object, IOContext io_context); extern bool pgstat_tracks_io_op(BackendType bktype, IOObject io_object, IOContext io_context, IOOp io_op); +extern void pgstat_create_backend_stat(ProcNumber procnum); /* diff --git a/src/include/utils/pgstat_internal.h b/src/include/utils/pgstat_internal.h index 61b2e1f96b..44a00d4779 100644 --- a/src/include/utils/pgstat_internal.h +++ b/src/include/utils/pgstat_internal.h @@ -194,6 +194,11 @@ typedef struct PgStat_KindInfo */ bool accessed_across_databases:1; + /* + * Do serialize or not this kind of stats. + */ + bool to_serialize:1; + /* * The size of an entry in the shared stats hash table (pointed to by * PgStatShared_HashEntry->body). For fixed-numbered statistics, this is @@ -428,6 +433,11 @@ typedef struct PgStatShared_ReplSlot PgStat_StatReplSlotEntry stats; } PgStatShared_ReplSlot; +typedef struct PgStatShared_Backend +{ + PgStatShared_Common header; + PgStat_Backend stats; +} PgStatShared_Backend; /* * Central shared memory entry for the cumulative stats system. @@ -630,9 +640,12 @@ extern void pgstat_flush_io(bool nowait); extern bool pgstat_io_have_pending_cb(void); extern bool pgstat_io_flush_cb(bool nowait); +extern bool pgstat_per_backend_flush_cb(PgStat_EntryRef *entry_ref, bool nowait); extern void pgstat_io_init_shmem_cb(void *stats); extern void pgstat_io_reset_all_cb(TimestampTz ts); extern void pgstat_io_snapshot_cb(void); +extern PgStat_PendingIO *pgstat_prep_per_backend_pending(ProcNumber procnum); +extern void pgstat_backend_reset_timestamp_cb(PgStatShared_Common *header, TimestampTz ts); /* diff --git a/src/test/modules/injection_points/injection_stats.c b/src/test/modules/injection_points/injection_stats.c index d89d055913..ca4b6b6e9f 100644 --- a/src/test/modules/injection_points/injection_stats.c +++ b/src/test/modules/injection_points/injection_stats.c @@ -39,6 +39,7 @@ static bool injection_stats_flush_cb(PgStat_EntryRef *entry_ref, bool nowait); static const PgStat_KindInfo injection_stats = { .name = "injection_points", .fixed_amount = false, /* Bounded by the number of points */ + .to_serialize = true, /* Injection points are system-wide */ .accessed_across_databases = true, diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out index 2b47013f11..f92516b047 100644 --- a/src/test/regress/expected/rules.out +++ b/src/test/regress/expected/rules.out @@ -1398,6 +1398,25 @@ pg_matviews| SELECT n.nspname AS schemaname, LEFT JOIN pg_namespace n ON ((n.oid = c.relnamespace))) LEFT JOIN pg_tablespace t ON ((t.oid = c.reltablespace))) WHERE (c.relkind = 'm'::"char"); +pg_my_stat_io| SELECT backend_type, + object, + context, + reads, + read_time, + writes, + write_time, + writebacks, + writeback_time, + extends, + extend_time, + op_bytes, + hits, + evictions, + reuses, + fsyncs, + fsync_time, + stats_reset + FROM pg_stat_get_my_io() b(backend_type, object, context, reads, read_time, writes, write_time, writebacks, writeback_time, extends, extend_time, op_bytes, hits, evictions, reuses, fsyncs, fsync_time, stats_reset); pg_policies| SELECT n.nspname AS schemaname, c.relname AS tablename, pol.polname AS policyname, diff --git a/src/test/regress/expected/stats.out b/src/test/regress/expected/stats.out index 56771f83ed..846e693483 100644 --- a/src/test/regress/expected/stats.out +++ b/src/test/regress/expected/stats.out @@ -1249,7 +1249,7 @@ SELECT pg_stat_get_subscription_stats(NULL); (1 row) --- Test that the following operations are tracked in pg_stat_io: +-- Test that the following operations are tracked in pg_[my_]stat_io: -- - reads of target blocks into shared buffers -- - writes of shared buffers to permanent storage -- - extends of relations using shared buffers @@ -1261,9 +1261,14 @@ SELECT pg_stat_get_subscription_stats(NULL); -- extends. SELECT sum(extends) AS io_sum_shared_before_extends FROM pg_stat_io WHERE context = 'normal' AND object = 'relation' \gset +SELECT sum(extends) AS my_io_sum_shared_before_extends + FROM pg_my_stat_io WHERE context = 'normal' AND object = 'relation' \gset SELECT sum(writes) AS writes, sum(fsyncs) AS fsyncs FROM pg_stat_io WHERE object = 'relation' \gset io_sum_shared_before_ +SELECT sum(writes) AS writes, sum(fsyncs) AS fsyncs + FROM pg_my_stat_io + WHERE object = 'relation' \gset my_io_sum_shared_before_ CREATE TABLE test_io_shared(a int); INSERT INTO test_io_shared SELECT i FROM generate_series(1,100)i; SELECT pg_stat_force_next_flush(); @@ -1280,8 +1285,16 @@ SELECT :io_sum_shared_after_extends > :io_sum_shared_before_extends; t (1 row) +SELECT sum(extends) AS my_io_sum_shared_after_extends + FROM pg_my_stat_io WHERE context = 'normal' AND object = 'relation' \gset +SELECT :my_io_sum_shared_after_extends > :my_io_sum_shared_before_extends; + ?column? +---------- + t +(1 row) + -- After a checkpoint, there should be some additional IOCONTEXT_NORMAL writes --- and fsyncs. +-- and fsyncs in the global stats (not for the backend). -- See comment above for rationale for two explicit CHECKPOINTs. CHECKPOINT; CHECKPOINT; @@ -1301,6 +1314,23 @@ SELECT current_setting('fsync') = 'off' t (1 row) +SELECT sum(writes) AS writes, sum(fsyncs) AS fsyncs + FROM pg_my_stat_io + WHERE object = 'relation' \gset my_io_sum_shared_after_ +SELECT :my_io_sum_shared_after_writes >= :my_io_sum_shared_before_writes; + ?column? +---------- + t +(1 row) + +SELECT current_setting('fsync') = 'off' + OR (:my_io_sum_shared_after_fsyncs = :my_io_sum_shared_before_fsyncs + AND :my_io_sum_shared_after_fsyncs= 0); + ?column? +---------- + t +(1 row) + -- Change the tablespace so that the table is rewritten directly, then SELECT -- from it to cause it to be read back into shared buffers. SELECT sum(reads) AS io_sum_shared_before_reads @@ -1521,6 +1551,8 @@ SELECT pg_stat_have_stats('io', 0, 0); SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS io_stats_pre_reset FROM pg_stat_io \gset +SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS my_io_stats_pre_reset + FROM pg_my_stat_io \gset SELECT pg_stat_reset_shared('io'); pg_stat_reset_shared ---------------------- @@ -1535,6 +1567,30 @@ SELECT :io_stats_post_reset < :io_stats_pre_reset; t (1 row) +SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS my_io_stats_post_reset + FROM pg_my_stat_io \gset +-- pg_stat_reset_shared() did not reset backend IO stats +SELECT :my_io_stats_pre_reset <= :my_io_stats_post_reset; + ?column? +---------- + t +(1 row) + +-- but pg_stat_reset_single_backend_io_counters() does +SELECT pg_stat_reset_single_backend_io_counters(pg_backend_pid()); + pg_stat_reset_single_backend_io_counters +------------------------------------------ + +(1 row) + +SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS my_io_stats_post_backend_reset + FROM pg_my_stat_io \gset +SELECT :my_io_stats_pre_reset > :my_io_stats_post_backend_reset; + ?column? +---------- + t +(1 row) + -- test BRIN index doesn't block HOT update CREATE TABLE brin_hot ( id integer PRIMARY KEY, diff --git a/src/test/regress/sql/stats.sql b/src/test/regress/sql/stats.sql index 7147cc2f89..9cb14b7182 100644 --- a/src/test/regress/sql/stats.sql +++ b/src/test/regress/sql/stats.sql @@ -595,7 +595,7 @@ SELECT pg_stat_get_replication_slot(NULL); SELECT pg_stat_get_subscription_stats(NULL); --- Test that the following operations are tracked in pg_stat_io: +-- Test that the following operations are tracked in pg_[my_]stat_io: -- - reads of target blocks into shared buffers -- - writes of shared buffers to permanent storage -- - extends of relations using shared buffers @@ -609,18 +609,26 @@ SELECT pg_stat_get_subscription_stats(NULL); -- extends. SELECT sum(extends) AS io_sum_shared_before_extends FROM pg_stat_io WHERE context = 'normal' AND object = 'relation' \gset +SELECT sum(extends) AS my_io_sum_shared_before_extends + FROM pg_my_stat_io WHERE context = 'normal' AND object = 'relation' \gset SELECT sum(writes) AS writes, sum(fsyncs) AS fsyncs FROM pg_stat_io WHERE object = 'relation' \gset io_sum_shared_before_ +SELECT sum(writes) AS writes, sum(fsyncs) AS fsyncs + FROM pg_my_stat_io + WHERE object = 'relation' \gset my_io_sum_shared_before_ CREATE TABLE test_io_shared(a int); INSERT INTO test_io_shared SELECT i FROM generate_series(1,100)i; SELECT pg_stat_force_next_flush(); SELECT sum(extends) AS io_sum_shared_after_extends FROM pg_stat_io WHERE context = 'normal' AND object = 'relation' \gset SELECT :io_sum_shared_after_extends > :io_sum_shared_before_extends; +SELECT sum(extends) AS my_io_sum_shared_after_extends + FROM pg_my_stat_io WHERE context = 'normal' AND object = 'relation' \gset +SELECT :my_io_sum_shared_after_extends > :my_io_sum_shared_before_extends; -- After a checkpoint, there should be some additional IOCONTEXT_NORMAL writes --- and fsyncs. +-- and fsyncs in the global stats (not for the backend). -- See comment above for rationale for two explicit CHECKPOINTs. CHECKPOINT; CHECKPOINT; @@ -631,6 +639,14 @@ SELECT :io_sum_shared_after_writes > :io_sum_shared_before_writes; SELECT current_setting('fsync') = 'off' OR :io_sum_shared_after_fsyncs > :io_sum_shared_before_fsyncs; +SELECT sum(writes) AS writes, sum(fsyncs) AS fsyncs + FROM pg_my_stat_io + WHERE object = 'relation' \gset my_io_sum_shared_after_ +SELECT :my_io_sum_shared_after_writes >= :my_io_sum_shared_before_writes; +SELECT current_setting('fsync') = 'off' + OR (:my_io_sum_shared_after_fsyncs = :my_io_sum_shared_before_fsyncs + AND :my_io_sum_shared_after_fsyncs= 0); + -- Change the tablespace so that the table is rewritten directly, then SELECT -- from it to cause it to be read back into shared buffers. SELECT sum(reads) AS io_sum_shared_before_reads @@ -762,10 +778,21 @@ SELECT :io_sum_bulkwrite_strategy_extends_after > :io_sum_bulkwrite_strategy_ext SELECT pg_stat_have_stats('io', 0, 0); SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS io_stats_pre_reset FROM pg_stat_io \gset +SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS my_io_stats_pre_reset + FROM pg_my_stat_io \gset SELECT pg_stat_reset_shared('io'); SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS io_stats_post_reset FROM pg_stat_io \gset SELECT :io_stats_post_reset < :io_stats_pre_reset; +SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS my_io_stats_post_reset + FROM pg_my_stat_io \gset +-- pg_stat_reset_shared() did not reset backend IO stats +SELECT :my_io_stats_pre_reset <= :my_io_stats_post_reset; +-- but pg_stat_reset_single_backend_io_counters() does +SELECT pg_stat_reset_single_backend_io_counters(pg_backend_pid()); +SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS my_io_stats_post_backend_reset + FROM pg_my_stat_io \gset +SELECT :my_io_stats_pre_reset > :my_io_stats_post_backend_reset; -- test BRIN index doesn't block HOT update diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 1847bbfa95..4188d056ca 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2117,6 +2117,7 @@ PgFdwSamplingMethod PgFdwScanState PgIfAddrCallback PgStatShared_Archiver +PgStatShared_Backend PgStatShared_BgWriter PgStatShared_Checkpointer PgStatShared_Common @@ -2132,6 +2133,7 @@ PgStatShared_SLRU PgStatShared_Subscription PgStatShared_Wal PgStat_ArchiverStats +PgStat_Backend PgStat_BackendSubEntry PgStat_BgWriterStats PgStat_BktypeIO -- 2.34.1 --xszoq7AUxWDP9Wxf Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v5-0002-Merge-both-IO-stats-flush-callbacks.patch" ^ permalink raw reply [nested|flat] 25+ messages in thread
* [PATCH v6] per backend I/O statistics @ 2024-10-28 12:50 Bertrand Drouvot <[email protected]> 0 siblings, 0 replies; 25+ messages in thread From: Bertrand Drouvot @ 2024-10-28 12:50 UTC (permalink / raw) While pg_stat_io provides cluster-wide I/O statistics, this commit adds the ability to track and display per backend I/O statistics. It adds a new statistics kind, a new pg_my_stat_io view to display "my" backend I/O statistics, and 2 new functions: - pg_stat_reset_single_backend_io_counters() to be able to reset the I/O stats for a given backend pid. - pg_stat_get_backend_io() to retrieve I/O statistics for a given backend pid. The new KIND is named PGSTAT_KIND_PER_BACKEND as it could be used in the future to store other statistics (than the I/O ones) per backend. The new KIND is a variable-numbered one and has an automatic cap on the maximum number of entries (as its hash key contains the proc number). There is no need to write the per backend I/O stats to disk (no point to see stats for backends that do not exist anymore after a re-start), so using "write_to_file = false". Note that per backend I/O statistics are not collected for the checkpointer, the background writer, the startup process and the autovacuum launcher as those are already visible in pg_stat_io and there is only one of those. XXX: Bump catalog version needs to be done. --- doc/src/sgml/config.sgml | 4 +- doc/src/sgml/monitoring.sgml | 63 ++++++ src/backend/catalog/system_functions.sql | 2 + src/backend/catalog/system_views.sql | 22 +++ src/backend/utils/activity/Makefile | 1 + src/backend/utils/activity/backend_status.c | 4 + src/backend/utils/activity/meson.build | 1 + src/backend/utils/activity/pgstat.c | 19 +- src/backend/utils/activity/pgstat_backend.c | 176 +++++++++++++++++ src/backend/utils/activity/pgstat_io.c | 37 +++- src/backend/utils/adt/pgstatfuncs.c | 208 ++++++++++++++++++++ src/include/catalog/pg_proc.dat | 14 ++ src/include/pgstat.h | 32 ++- src/include/utils/pgstat_internal.h | 12 ++ src/test/regress/expected/rules.out | 19 ++ src/test/regress/expected/stats.out | 85 +++++++- src/test/regress/sql/stats.sql | 47 ++++- src/tools/pgindent/typedefs.list | 3 + 18 files changed, 729 insertions(+), 20 deletions(-) 11.6% doc/src/sgml/ 27.0% src/backend/utils/activity/ 22.4% src/backend/utils/adt/ 4.0% src/include/catalog/ 5.9% src/include/ 14.9% src/test/regress/expected/ 11.4% src/test/regress/sql/ diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index a84e60c09b..20a0862661 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -8365,7 +8365,9 @@ COPY postgres_log FROM '/full/path/to/logfile.csv' WITH csv; displayed in <link linkend="monitoring-pg-stat-database-view"> <structname>pg_stat_database</structname></link>, <link linkend="monitoring-pg-stat-io-view"> - <structname>pg_stat_io</structname></link>, in the output of + <structname>pg_stat_io</structname></link>, + <link linkend="monitoring-pg-my-stat-io-view"> + <structname>pg_my_stat_io</structname></link>, in the output of <xref linkend="sql-explain"/> when the <literal>BUFFERS</literal> option is used, in the output of <xref linkend="sql-vacuum"/> when the <literal>VERBOSE</literal> option is used, by autovacuum diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml index 840d7f8161..8e7a7a0f36 100644 --- a/doc/src/sgml/monitoring.sgml +++ b/doc/src/sgml/monitoring.sgml @@ -488,6 +488,16 @@ postgres 27093 0.0 0.0 30096 2752 ? Ss 11:34 0:00 postgres: ser </entry> </row> + <row> + <entry><structname>pg_my_stat_io</structname><indexterm><primary>pg_my_stat_io</primary></indexterm></entry> + <entry> + One row for each combination of context and target object containing + my backend I/O statistics. + See <link linkend="monitoring-pg-my-stat-io-view"> + <structname>pg_my_stat_io</structname></link> for details. + </entry> + </row> + <row> <entry><structname>pg_stat_replication_slots</structname><indexterm><primary>pg_stat_replication_slots</primary></indexterm></entry> <entry>One row per replication slot, showing statistics about the @@ -2946,7 +2956,23 @@ description | Waiting for a newly initialized WAL file to reach durable storage </para> </note> + </sect2> + + <sect2 id="monitoring-pg-my-stat-io-view"> + <title><structname>pg_my_stat_io</structname></title> + + <indexterm> + <primary>pg_my_stat_io</primary> + </indexterm> + <para> + The <structname>pg_my_stat_io</structname> view will contain one row for each + combination of target I/O object and I/O context, showing + my backend I/O statistics. Combinations which do not make sense are + omitted. The fields are exactly the same as the ones in the <link + linkend="monitoring-pg-stat-io-view"> <structname>pg_stat_io</structname></link> + view. + </para> </sect2> @@ -4790,6 +4816,25 @@ description | Waiting for a newly initialized WAL file to reach durable storage </para></entry> </row> + <row> + <entry role="func_table_entry"><para role="func_signature"> + <indexterm> + <primary>pg_stat_get_backend_io</primary> + </indexterm> + <function>pg_stat_get_backend_io</function> ( <type>integer</type> ) + <returnvalue>setof record</returnvalue> + </para> + <para> + Returns I/O statistics about the backend with the specified + process ID. The output fields are exactly the same as the ones in the + <link linkend="monitoring-pg-stat-io-view"> <structname>pg_stat_io</structname></link> + view. The function does not return I/O statistics for the checkpointer, + the background writer, the startup process and the autovacuum launcher + as they are already visible in the <link linkend="monitoring-pg-stat-io-view"> <structname>pg_stat_io</structname></link> + view and there is only one of those. + </para></entry> + </row> + <row> <entry role="func_table_entry"><para role="func_signature"> <indexterm> @@ -4971,6 +5016,24 @@ description | Waiting for a newly initialized WAL file to reach durable storage </para></entry> </row> + <row> + <entry role="func_table_entry"><para role="func_signature"> + <indexterm> + <primary>pg_stat_reset_single_backend_io_counters</primary> + </indexterm> + <function>pg_stat_reset_single_backend_io_counters</function> ( <type>integer</type> ) + <returnvalue>void</returnvalue> + </para> + <para> + Resets I/O statistics for a single backend with the specified process ID + to zero. + </para> + <para> + This function is restricted to superusers by default, but other users + can be granted EXECUTE to run the function. + </para></entry> + </row> + <row> <entry role="func_table_entry"><para role="func_signature"> <indexterm> diff --git a/src/backend/catalog/system_functions.sql b/src/backend/catalog/system_functions.sql index c51dfca802..1b21b242cd 100644 --- a/src/backend/catalog/system_functions.sql +++ b/src/backend/catalog/system_functions.sql @@ -711,6 +711,8 @@ REVOKE EXECUTE ON FUNCTION pg_stat_reset_single_table_counters(oid) FROM public; REVOKE EXECUTE ON FUNCTION pg_stat_reset_single_function_counters(oid) FROM public; +REVOKE EXECUTE ON FUNCTION pg_stat_reset_single_backend_io_counters(integer) FROM public; + REVOKE EXECUTE ON FUNCTION pg_stat_reset_replication_slot(text) FROM public; REVOKE EXECUTE ON FUNCTION pg_stat_have_stats(text, oid, int8) FROM public; diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index da9a8fe99f..fce3b791bb 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1172,6 +1172,28 @@ SELECT b.stats_reset FROM pg_stat_get_io() b; +CREATE VIEW pg_my_stat_io AS +SELECT + b.backend_type, + b.object, + b.context, + b.reads, + b.read_time, + b.writes, + b.write_time, + b.writebacks, + b.writeback_time, + b.extends, + b.extend_time, + b.op_bytes, + b.hits, + b.evictions, + b.reuses, + b.fsyncs, + b.fsync_time, + b.stats_reset +FROM pg_stat_get_backend_io(NULL) b; + CREATE VIEW pg_stat_wal AS SELECT w.wal_records, diff --git a/src/backend/utils/activity/Makefile b/src/backend/utils/activity/Makefile index b9fd66ea17..24b64a2742 100644 --- a/src/backend/utils/activity/Makefile +++ b/src/backend/utils/activity/Makefile @@ -20,6 +20,7 @@ OBJS = \ backend_status.o \ pgstat.o \ pgstat_archiver.o \ + pgstat_backend.o \ pgstat_bgwriter.o \ pgstat_checkpointer.o \ pgstat_database.o \ diff --git a/src/backend/utils/activity/backend_status.c b/src/backend/utils/activity/backend_status.c index bdb3a296ca..249439b990 100644 --- a/src/backend/utils/activity/backend_status.c +++ b/src/backend/utils/activity/backend_status.c @@ -249,6 +249,10 @@ pgstat_beinit(void) Assert(MyProcNumber >= 0 && MyProcNumber < NumBackendStatSlots); MyBEEntry = &BackendStatusArray[MyProcNumber]; + /* Create the per-backend statistics entry */ + if (pgstat_tracks_per_backend_bktype(MyBackendType)) + pgstat_create_backend_stat(MyProcNumber); + /* Set up a process-exit hook to clean up */ on_shmem_exit(pgstat_beshutdown_hook, 0); } diff --git a/src/backend/utils/activity/meson.build b/src/backend/utils/activity/meson.build index f73c22905c..380d3dd70c 100644 --- a/src/backend/utils/activity/meson.build +++ b/src/backend/utils/activity/meson.build @@ -5,6 +5,7 @@ backend_sources += files( 'backend_status.c', 'pgstat.c', 'pgstat_archiver.c', + 'pgstat_backend.c', 'pgstat_bgwriter.c', 'pgstat_checkpointer.c', 'pgstat_database.c', diff --git a/src/backend/utils/activity/pgstat.c b/src/backend/utils/activity/pgstat.c index 6f8d237826..1449d6a3f4 100644 --- a/src/backend/utils/activity/pgstat.c +++ b/src/backend/utils/activity/pgstat.c @@ -77,6 +77,7 @@ * * Each statistics kind is handled in a dedicated file: * - pgstat_archiver.c + * - pgstat_backend.c * - pgstat_bgwriter.c * - pgstat_checkpointer.c * - pgstat_database.c @@ -358,6 +359,22 @@ static const PgStat_KindInfo pgstat_kind_builtin_infos[PGSTAT_KIND_BUILTIN_SIZE] .reset_timestamp_cb = pgstat_subscription_reset_timestamp_cb, }, + [PGSTAT_KIND_PER_BACKEND] = { + .name = "per-backend", + + .fixed_amount = false, + .write_to_file = false, + + .accessed_across_databases = true, + + .shared_size = sizeof(PgStatShared_Backend), + .shared_data_off = offsetof(PgStatShared_Backend, stats), + .shared_data_len = sizeof(((PgStatShared_Backend *) 0)->stats), + .pending_size = sizeof(PgStat_BackendPendingIO), + + .flush_pending_cb = pgstat_per_backend_flush_cb, + .reset_timestamp_cb = pgstat_backend_reset_timestamp_cb, + }, /* stats for fixed-numbered (mostly 1) objects */ @@ -768,7 +785,7 @@ pgstat_report_stat(bool force) partial_flush = false; - /* flush database / relation / function / ... stats */ + /* flush database / relation / function / backend / ... stats */ partial_flush |= pgstat_flush_pending_entries(nowait); /* flush of fixed-numbered stats */ diff --git a/src/backend/utils/activity/pgstat_backend.c b/src/backend/utils/activity/pgstat_backend.c new file mode 100644 index 0000000000..f079f15ff2 --- /dev/null +++ b/src/backend/utils/activity/pgstat_backend.c @@ -0,0 +1,176 @@ +/* ------------------------------------------------------------------------- + * + * pgstat_backend.c + * Implementation of per-backend statistics. + * + * This file contains the implementation of per-backend statistics. It is kept + * separate from pgstat.c to enforce the line between the statistics access / + * storage implementation and the details about individual types of statistics. + * + * Copyright (c) 2024, PostgreSQL Global Development Group + * + * IDENTIFICATION + * src/backend/utils/activity/pgstat_backend.c + * ------------------------------------------------------------------------- + */ + +#include "postgres.h" + +#include "utils/pgstat_internal.h" + +/* + * Returns my backend IO stats. + */ +PgStat_Backend * +pgstat_fetch_my_stat_io(void) +{ + return (PgStat_Backend *) + pgstat_fetch_entry(PGSTAT_KIND_PER_BACKEND, InvalidOid, MyProcNumber); +} + +/* + * Returns other backend's IO stats. + */ +PgStat_Backend * +pgstat_fetch_proc_stat_io(ProcNumber procNumber) +{ + PgStat_Backend *backend_entry; + + backend_entry = (PgStat_Backend *) pgstat_fetch_entry(PGSTAT_KIND_PER_BACKEND, + InvalidOid, procNumber); + + return backend_entry; +} + +/* + * Flush out locally pending backend statistics + * + * If no stats have been recorded, this function returns false. + */ +bool +pgstat_per_backend_flush_cb(PgStat_EntryRef *entry_ref, bool nowait) +{ + PgStatShared_Backend *shbackendioent; + PgStat_BackendPendingIO *pendingent; + PgStat_BktypeIO *bktype_shstats; + + if (!pgstat_lock_entry(entry_ref, nowait)) + return false; + + shbackendioent = (PgStatShared_Backend *) entry_ref->shared_stats; + bktype_shstats = &shbackendioent->stats.stats; + pendingent = (PgStat_BackendPendingIO *) entry_ref->pending; + + for (int io_object = 0; io_object < IOOBJECT_NUM_TYPES; io_object++) + { + for (int io_context = 0; io_context < IOCONTEXT_NUM_TYPES; io_context++) + { + for (int io_op = 0; io_op < IOOP_NUM_TYPES; io_op++) + { + instr_time time; + + bktype_shstats->counts[io_object][io_context][io_op] += + pendingent->counts[io_object][io_context][io_op]; + + time = pendingent->pending_times[io_object][io_context][io_op]; + + bktype_shstats->times[io_object][io_context][io_op] += + INSTR_TIME_GET_MICROSEC(time); + } + } + } + + pgstat_unlock_entry(entry_ref); + + return true; +} + +/* + * Create the per-backend statistics entry for procnum. + */ +void +pgstat_create_backend_stat(ProcNumber procnum) +{ + PgStat_EntryRef *entry_ref; + PgStatShared_Backend *shstatent; + + entry_ref = pgstat_prep_pending_entry(PGSTAT_KIND_PER_BACKEND, InvalidOid, + procnum, NULL); + + shstatent = (PgStatShared_Backend *) entry_ref->shared_stats; + + /* + * NB: need to accept that there might be stats from an older backend, + * e.g. if we previously used this proc number. + */ + memset(&shstatent->stats, 0, sizeof(shstatent->stats)); +} + +/* + * Find or create a local PgStat_BackendPendingIO entry for procnum. + */ +PgStat_BackendPendingIO * +pgstat_prep_per_backend_pending(ProcNumber procnum) +{ + PgStat_EntryRef *entry_ref; + + entry_ref = pgstat_prep_pending_entry(PGSTAT_KIND_PER_BACKEND, InvalidOid, + procnum, NULL); + + return entry_ref->pending; +} + +/* + * per-backend statistics are not collected for all BackendTypes. + * + * The following BackendTypes do not participate in the per-backend stats + * subsystem: + * - The same and for the same reasons as in pgstat_tracks_io_bktype(). + * - B_BG_WRITER, B_CHECKPOINTER, B_STARTUP and B_AUTOVAC_LAUNCHER because their + * I/O stats are already visible in pg_stat_io and there is only one of those. + * + * Function returns true if BackendType participates in the per-backend stats + * subsystem for IO and false if it does not. + * + * When adding a new BackendType, also consider adding relevant restrictions to + * pgstat_tracks_io_object() and pgstat_tracks_io_op(). + */ +bool +pgstat_tracks_per_backend_bktype(BackendType bktype) +{ + /* + * List every type so that new backend types trigger a warning about + * needing to adjust this switch. + */ + switch (bktype) + { + case B_INVALID: + case B_AUTOVAC_LAUNCHER: + case B_DEAD_END_BACKEND: + case B_ARCHIVER: + case B_LOGGER: + case B_WAL_RECEIVER: + case B_WAL_WRITER: + case B_WAL_SUMMARIZER: + case B_BG_WRITER: + case B_CHECKPOINTER: + case B_STARTUP: + return false; + + case B_AUTOVAC_WORKER: + case B_BACKEND: + case B_BG_WORKER: + case B_STANDALONE_BACKEND: + case B_SLOTSYNC_WORKER: + case B_WAL_SENDER: + return true; + } + + return false; +} + +void +pgstat_backend_reset_timestamp_cb(PgStatShared_Common *header, TimestampTz ts) +{ + ((PgStatShared_Backend *) header)->stats.stat_reset_timestamp = ts; +} diff --git a/src/backend/utils/activity/pgstat_io.c b/src/backend/utils/activity/pgstat_io.c index f9883af2b3..1af5b75df6 100644 --- a/src/backend/utils/activity/pgstat_io.c +++ b/src/backend/utils/activity/pgstat_io.c @@ -20,13 +20,7 @@ #include "storage/bufmgr.h" #include "utils/pgstat_internal.h" - -typedef struct PgStat_PendingIO -{ - PgStat_Counter counts[IOOBJECT_NUM_TYPES][IOCONTEXT_NUM_TYPES][IOOP_NUM_TYPES]; - instr_time pending_times[IOOBJECT_NUM_TYPES][IOCONTEXT_NUM_TYPES][IOOP_NUM_TYPES]; -} PgStat_PendingIO; - +typedef PgStat_BackendPendingIO PgStat_PendingIO; static PgStat_PendingIO PendingIOStats; static bool have_iostats = false; @@ -87,6 +81,14 @@ pgstat_count_io_op_n(IOObject io_object, IOContext io_context, IOOp io_op, uint3 Assert((unsigned int) io_op < IOOP_NUM_TYPES); Assert(pgstat_tracks_io_op(MyBackendType, io_object, io_context, io_op)); + if (pgstat_tracks_per_backend_bktype(MyBackendType)) + { + PgStat_PendingIO *entry_ref; + + entry_ref = pgstat_prep_per_backend_pending(MyProcNumber); + entry_ref->counts[io_object][io_context][io_op] += cnt; + } + PendingIOStats.counts[io_object][io_context][io_op] += cnt; have_iostats = true; @@ -122,6 +124,7 @@ void pgstat_count_io_op_time(IOObject io_object, IOContext io_context, IOOp io_op, instr_time start_time, uint32 cnt) { + if (track_io_timing) { instr_time io_time; @@ -148,6 +151,15 @@ pgstat_count_io_op_time(IOObject io_object, IOContext io_context, IOOp io_op, INSTR_TIME_ADD(PendingIOStats.pending_times[io_object][io_context][io_op], io_time); + + if (pgstat_tracks_per_backend_bktype(MyBackendType)) + { + PgStat_PendingIO *entry_ref; + + entry_ref = pgstat_prep_per_backend_pending(MyProcNumber); + INSTR_TIME_ADD(entry_ref->pending_times[io_object][io_context][io_op], + io_time); + } } pgstat_count_io_op_n(io_object, io_context, io_op, cnt); @@ -171,12 +183,21 @@ pgstat_io_have_pending_cb(void) } /* - * Simpler wrapper of pgstat_io_flush_cb() + * Simpler wrapper of pgstat_io_flush_cb() and pgstat_per_backend_flush_cb(). */ void pgstat_flush_io(bool nowait) { + PgStat_EntryRef *entry_ref; + (void) pgstat_io_flush_cb(nowait); + + if (pgstat_tracks_per_backend_bktype(MyBackendType)) + { + entry_ref = pgstat_get_entry_ref(PGSTAT_KIND_PER_BACKEND, InvalidOid, + MyProcNumber, false, NULL); + (void) pgstat_per_backend_flush_cb(entry_ref, nowait); + } } /* diff --git a/src/backend/utils/adt/pgstatfuncs.c b/src/backend/utils/adt/pgstatfuncs.c index 60a397dc56..0302970246 100644 --- a/src/backend/utils/adt/pgstatfuncs.c +++ b/src/backend/utils/adt/pgstatfuncs.c @@ -1480,6 +1480,165 @@ pg_stat_get_io(PG_FUNCTION_ARGS) return (Datum) 0; } +Datum +pg_stat_get_backend_io(PG_FUNCTION_ARGS) +{ + ReturnSetInfo *rsinfo; + PgStat_Backend *backend_stats; + Datum bktype_desc; + PgStat_BktypeIO *bktype_stats; + BackendType bktype; + Datum reset_time; + int num_backends = pgstat_fetch_stat_numbackends(); + int curr_backend; + int pid; + + /* just to keep compiler quiet */ + bktype = B_INVALID; + + InitMaterializedSRF(fcinfo, 0); + rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; + + if (PG_ARGISNULL(0) || PG_GETARG_INT32(0) == MyProcPid) + { + backend_stats = pgstat_fetch_my_stat_io(); + pid = MyProcPid; + bktype = MyBackendType; + } + else + { + PGPROC *proc; + ProcNumber procNumber; + + pid = PG_GETARG_INT32(0); + proc = BackendPidGetProc(pid); + + /* + * Maybe an auxiliary process? That should not be possible, due to + * pgstat_tracks_per_backend_bktype() though. + */ + if (proc == NULL) + proc = AuxiliaryPidGetProc(pid); + + if (proc != NULL) + { + procNumber = GetNumberFromPGProc(proc); + backend_stats = pgstat_fetch_proc_stat_io(procNumber); + + if (!backend_stats) + return (Datum) 0; + } + else + return (Datum) 0; + } + + /* Look for the backend type */ + for (curr_backend = 1; curr_backend <= num_backends; curr_backend++) + { + LocalPgBackendStatus *local_beentry; + PgBackendStatus *beentry; + + /* Get the next one in the list */ + local_beentry = pgstat_get_local_beentry_by_index(curr_backend); + beentry = &local_beentry->backendStatus; + + /* looking for specific PID, ignore all the others */ + if (beentry->st_procpid != pid) + continue; + + bktype = beentry->st_backendType; + break; + } + + bktype_desc = CStringGetTextDatum(GetBackendTypeDesc(bktype)); + bktype_stats = &backend_stats->stats; + reset_time = TimestampTzGetDatum(backend_stats->stat_reset_timestamp); + + /* + * In Assert builds, we can afford an extra loop through all of the + * counters checking that only expected stats are non-zero, since it keeps + * the non-Assert code cleaner. + */ + Assert(pgstat_bktype_io_stats_valid(bktype_stats, bktype)); + + for (int io_obj = 0; io_obj < IOOBJECT_NUM_TYPES; io_obj++) + { + const char *obj_name = pgstat_get_io_object_name(io_obj); + + for (int io_context = 0; io_context < IOCONTEXT_NUM_TYPES; io_context++) + { + const char *context_name = pgstat_get_io_context_name(io_context); + + Datum values[IO_NUM_COLUMNS] = {0}; + bool nulls[IO_NUM_COLUMNS] = {0}; + + /* + * Some combinations of BackendType, IOObject, and IOContext are + * not valid for any type of IOOp. In such cases, omit the entire + * row from the view. + */ + if (!pgstat_tracks_io_object(bktype, io_obj, io_context)) + continue; + + values[IO_COL_BACKEND_TYPE] = bktype_desc; + values[IO_COL_CONTEXT] = CStringGetTextDatum(context_name); + values[IO_COL_OBJECT] = CStringGetTextDatum(obj_name); + if (backend_stats->stat_reset_timestamp != 0) + values[IO_COL_RESET_TIME] = reset_time; + else + nulls[IO_COL_RESET_TIME] = true; + + /* + * Hard-code this to the value of BLCKSZ for now. Future values + * could include XLOG_BLCKSZ, once WAL IO is tracked, and constant + * multipliers, once non-block-oriented IO (e.g. temporary file + * IO) is tracked. + */ + values[IO_COL_CONVERSION] = Int64GetDatum(BLCKSZ); + + for (int io_op = 0; io_op < IOOP_NUM_TYPES; io_op++) + { + int op_idx = pgstat_get_io_op_index(io_op); + int time_idx = pgstat_get_io_time_index(io_op); + + /* + * Some combinations of BackendType and IOOp, of IOContext and + * IOOp, and of IOObject and IOOp are not tracked. Set these + * cells in the view NULL. + */ + if (pgstat_tracks_io_op(bktype, io_obj, io_context, io_op)) + { + PgStat_Counter count = + bktype_stats->counts[io_obj][io_context][io_op]; + + values[op_idx] = Int64GetDatum(count); + } + else + nulls[op_idx] = true; + + /* not every operation is timed */ + if (time_idx == IO_COL_INVALID) + continue; + + if (!nulls[op_idx]) + { + PgStat_Counter time = + bktype_stats->times[io_obj][io_context][io_op]; + + values[time_idx] = Float8GetDatum(pg_stat_us_to_ms(time)); + } + else + nulls[time_idx] = true; + } + + tuplestore_putvalues(rsinfo->setResult, rsinfo->setDesc, + values, nulls); + } + } + + return (Datum) 0; +} + /* * Returns statistics of WAL activity */ @@ -1785,6 +1944,55 @@ pg_stat_reset_single_function_counters(PG_FUNCTION_ARGS) PG_RETURN_VOID(); } +Datum +pg_stat_reset_single_backend_io_counters(PG_FUNCTION_ARGS) +{ + PGPROC *proc; + BackendType bktype; + int backend_pid = PG_GETARG_INT32(0); + int num_backends = pgstat_fetch_stat_numbackends(); + int curr_backend; + + proc = BackendPidGetProc(backend_pid); + + /* + * Maybe an auxiliary process? That should not be possible, due to + * pgstat_tracks_per_backend_bktype() though. + */ + if (proc == NULL) + proc = AuxiliaryPidGetProc(backend_pid); + + /* Check that IO stats are collected for this backend type */ + if (proc) + { + for (curr_backend = 1; curr_backend <= num_backends; curr_backend++) + { + LocalPgBackendStatus *local_beentry; + PgBackendStatus *beentry; + + /* Get the next one in the list */ + local_beentry = pgstat_get_local_beentry_by_index(curr_backend); + beentry = &local_beentry->backendStatus; + + /* looking for specific PID, ignore all the others */ + if (beentry->st_procpid != backend_pid) + continue; + + bktype = beentry->st_backendType; + + if (!pgstat_tracks_per_backend_bktype(bktype)) + PG_RETURN_VOID(); + else + { + pgstat_reset(PGSTAT_KIND_PER_BACKEND, InvalidOid, GetNumberFromPGProc(proc)); + break; + } + } + } + + PG_RETURN_VOID(); +} + /* Reset SLRU counters (a specific one or all of them). */ Datum pg_stat_reset_slru(PG_FUNCTION_ARGS) diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index cbbe8acd38..6e5c2082a7 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -5913,6 +5913,15 @@ proargnames => '{backend_type,object,context,reads,read_time,writes,write_time,writebacks,writeback_time,extends,extend_time,op_bytes,hits,evictions,reuses,fsyncs,fsync_time,stats_reset}', prosrc => 'pg_stat_get_io' }, +{ oid => '8806', descr => 'statistics: per backend IO statistics', + proname => 'pg_stat_get_backend_io', prorows => '5', proretset => 't', + provolatile => 'v', proparallel => 'r', prorettype => 'record', + proargtypes => 'int4', proisstrict => 'f', + proallargtypes => '{int4,text,text,text,int8,float8,int8,float8,int8,float8,int8,float8,int8,int8,int8,int8,int8,float8,timestamptz}', + proargmodes => '{i,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o}', + proargnames => '{backend_pid,backend_type,object,context,reads,read_time,writes,write_time,writebacks,writeback_time,extends,extend_time,op_bytes,hits,evictions,reuses,fsyncs,fsync_time,stats_reset}', + prosrc => 'pg_stat_get_backend_io' }, + { oid => '1136', descr => 'statistics: information about WAL activity', proname => 'pg_stat_get_wal', proisstrict => 'f', provolatile => 's', proparallel => 'r', prorettype => 'record', proargtypes => '', @@ -6052,6 +6061,11 @@ proname => 'pg_stat_reset_single_function_counters', provolatile => 'v', prorettype => 'void', proargtypes => 'oid', prosrc => 'pg_stat_reset_single_function_counters' }, +{ oid => '9987', + descr => 'statistics: reset collected IO statistics for a single backend', + proname => 'pg_stat_reset_single_backend_io_counters', provolatile => 'v', + prorettype => 'void', proargtypes => 'int4', + prosrc => 'pg_stat_reset_single_backend_io_counters' }, { oid => '2307', descr => 'statistics: reset collected statistics for a single SLRU', proname => 'pg_stat_reset_slru', proisstrict => 'f', provolatile => 'v', diff --git a/src/include/pgstat.h b/src/include/pgstat.h index 59c28b4aca..c5151c8e6f 100644 --- a/src/include/pgstat.h +++ b/src/include/pgstat.h @@ -49,14 +49,15 @@ #define PGSTAT_KIND_FUNCTION 3 /* per-function statistics */ #define PGSTAT_KIND_REPLSLOT 4 /* per-slot statistics */ #define PGSTAT_KIND_SUBSCRIPTION 5 /* per-subscription statistics */ +#define PGSTAT_KIND_PER_BACKEND 6 /* stats for fixed-numbered objects */ -#define PGSTAT_KIND_ARCHIVER 6 -#define PGSTAT_KIND_BGWRITER 7 -#define PGSTAT_KIND_CHECKPOINTER 8 -#define PGSTAT_KIND_IO 9 -#define PGSTAT_KIND_SLRU 10 -#define PGSTAT_KIND_WAL 11 +#define PGSTAT_KIND_ARCHIVER 7 +#define PGSTAT_KIND_BGWRITER 8 +#define PGSTAT_KIND_CHECKPOINTER 9 +#define PGSTAT_KIND_IO 10 +#define PGSTAT_KIND_SLRU 11 +#define PGSTAT_KIND_WAL 12 #define PGSTAT_KIND_BUILTIN_MIN PGSTAT_KIND_DATABASE #define PGSTAT_KIND_BUILTIN_MAX PGSTAT_KIND_WAL @@ -347,12 +348,23 @@ typedef struct PgStat_BktypeIO PgStat_Counter times[IOOBJECT_NUM_TYPES][IOCONTEXT_NUM_TYPES][IOOP_NUM_TYPES]; } PgStat_BktypeIO; +typedef struct PgStat_BackendPendingIO +{ + PgStat_Counter counts[IOOBJECT_NUM_TYPES][IOCONTEXT_NUM_TYPES][IOOP_NUM_TYPES]; + instr_time pending_times[IOOBJECT_NUM_TYPES][IOCONTEXT_NUM_TYPES][IOOP_NUM_TYPES]; +} PgStat_BackendPendingIO; + typedef struct PgStat_IO { TimestampTz stat_reset_timestamp; PgStat_BktypeIO stats[BACKEND_NUM_TYPES]; } PgStat_IO; +typedef struct PgStat_Backend +{ + TimestampTz stat_reset_timestamp; + PgStat_BktypeIO stats; +} PgStat_Backend; typedef struct PgStat_StatDBEntry { @@ -534,6 +546,14 @@ extern bool pgstat_have_entry(PgStat_Kind kind, Oid dboid, uint64 objid); extern void pgstat_report_archiver(const char *xlog, bool failed); extern PgStat_ArchiverStats *pgstat_fetch_stat_archiver(void); +/* + * Functions in pgstat_backend.c + */ + +extern PgStat_Backend *pgstat_fetch_my_stat_io(void); +extern PgStat_Backend *pgstat_fetch_proc_stat_io(ProcNumber procNumber); +extern bool pgstat_tracks_per_backend_bktype(BackendType bktype); +extern void pgstat_create_backend_stat(ProcNumber procnum); /* * Functions in pgstat_bgwriter.c diff --git a/src/include/utils/pgstat_internal.h b/src/include/utils/pgstat_internal.h index 7338bc1e28..625bb55864 100644 --- a/src/include/utils/pgstat_internal.h +++ b/src/include/utils/pgstat_internal.h @@ -450,6 +450,11 @@ typedef struct PgStatShared_ReplSlot PgStat_StatReplSlotEntry stats; } PgStatShared_ReplSlot; +typedef struct PgStatShared_Backend +{ + PgStatShared_Common header; + PgStat_Backend stats; +} PgStatShared_Backend; /* * Central shared memory entry for the cumulative stats system. @@ -604,6 +609,13 @@ extern void pgstat_archiver_init_shmem_cb(void *stats); extern void pgstat_archiver_reset_all_cb(TimestampTz ts); extern void pgstat_archiver_snapshot_cb(void); +/* + * Functions in pgstat_backend.c + */ + +extern PgStat_BackendPendingIO *pgstat_prep_per_backend_pending(ProcNumber procnum); +extern bool pgstat_per_backend_flush_cb(PgStat_EntryRef *entry_ref, bool nowait); +extern void pgstat_backend_reset_timestamp_cb(PgStatShared_Common *header, TimestampTz ts); /* * Functions in pgstat_bgwriter.c diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out index 3014d047fe..3922675bbf 100644 --- a/src/test/regress/expected/rules.out +++ b/src/test/regress/expected/rules.out @@ -1398,6 +1398,25 @@ pg_matviews| SELECT n.nspname AS schemaname, LEFT JOIN pg_namespace n ON ((n.oid = c.relnamespace))) LEFT JOIN pg_tablespace t ON ((t.oid = c.reltablespace))) WHERE (c.relkind = 'm'::"char"); +pg_my_stat_io| SELECT backend_type, + object, + context, + reads, + read_time, + writes, + write_time, + writebacks, + writeback_time, + extends, + extend_time, + op_bytes, + hits, + evictions, + reuses, + fsyncs, + fsync_time, + stats_reset + FROM pg_stat_get_backend_io(NULL::integer) b(backend_type, object, context, reads, read_time, writes, write_time, writebacks, writeback_time, extends, extend_time, op_bytes, hits, evictions, reuses, fsyncs, fsync_time, stats_reset); pg_policies| SELECT n.nspname AS schemaname, c.relname AS tablename, pol.polname AS policyname, diff --git a/src/test/regress/expected/stats.out b/src/test/regress/expected/stats.out index 56771f83ed..7f2fd873b9 100644 --- a/src/test/regress/expected/stats.out +++ b/src/test/regress/expected/stats.out @@ -1249,7 +1249,7 @@ SELECT pg_stat_get_subscription_stats(NULL); (1 row) --- Test that the following operations are tracked in pg_stat_io: +-- Test that the following operations are tracked in pg_[my_]stat_io: -- - reads of target blocks into shared buffers -- - writes of shared buffers to permanent storage -- - extends of relations using shared buffers @@ -1259,11 +1259,24 @@ SELECT pg_stat_get_subscription_stats(NULL); -- be sure of the state of shared buffers at the point the test is run. -- Create a regular table and insert some data to generate IOCONTEXT_NORMAL -- extends. +SELECT pid AS checkpointer_pid FROM pg_stat_activity + WHERE backend_type = 'checkpointer' \gset SELECT sum(extends) AS io_sum_shared_before_extends FROM pg_stat_io WHERE context = 'normal' AND object = 'relation' \gset +SELECT sum(extends) AS my_io_sum_shared_before_extends + FROM pg_my_stat_io WHERE context = 'normal' AND object = 'relation' \gset +SELECT sum(extends) AS backend_io_sum_shared_before_extends + FROM pg_stat_get_backend_io(pg_backend_pid()) + WHERE context = 'normal' AND object = 'relation' \gset SELECT sum(writes) AS writes, sum(fsyncs) AS fsyncs FROM pg_stat_io WHERE object = 'relation' \gset io_sum_shared_before_ +SELECT sum(writes) AS writes, sum(fsyncs) AS fsyncs + FROM pg_my_stat_io + WHERE object = 'relation' \gset my_io_sum_shared_before_ +SELECT sum(writes) AS writes, sum(fsyncs) AS fsyncs + FROM pg_stat_get_backend_io(pg_backend_pid()) + WHERE object = 'relation' \gset backend_io_sum_shared_before_ CREATE TABLE test_io_shared(a int); INSERT INTO test_io_shared SELECT i FROM generate_series(1,100)i; SELECT pg_stat_force_next_flush(); @@ -1280,8 +1293,25 @@ SELECT :io_sum_shared_after_extends > :io_sum_shared_before_extends; t (1 row) +SELECT sum(extends) AS my_io_sum_shared_after_extends + FROM pg_my_stat_io WHERE context = 'normal' AND object = 'relation' \gset +SELECT :my_io_sum_shared_after_extends > :my_io_sum_shared_before_extends; + ?column? +---------- + t +(1 row) + +SELECT sum(extends) AS backend_io_sum_shared_after_extends + FROM pg_stat_get_backend_io(pg_backend_pid()) + WHERE context = 'normal' AND object = 'relation' \gset +SELECT :backend_io_sum_shared_after_extends > :backend_io_sum_shared_before_extends; + ?column? +---------- + t +(1 row) + -- After a checkpoint, there should be some additional IOCONTEXT_NORMAL writes --- and fsyncs. +-- and fsyncs in the global stats (not for the backend). -- See comment above for rationale for two explicit CHECKPOINTs. CHECKPOINT; CHECKPOINT; @@ -1301,6 +1331,31 @@ SELECT current_setting('fsync') = 'off' t (1 row) +SELECT sum(writes) AS writes, sum(fsyncs) AS fsyncs + FROM pg_my_stat_io + WHERE object = 'relation' \gset my_io_sum_shared_after_ +SELECT :my_io_sum_shared_after_writes >= :my_io_sum_shared_before_writes; + ?column? +---------- + t +(1 row) + +SELECT current_setting('fsync') = 'off' + OR (:my_io_sum_shared_after_fsyncs = :my_io_sum_shared_before_fsyncs + AND :my_io_sum_shared_after_fsyncs= 0); + ?column? +---------- + t +(1 row) + +-- Don't return any rows if querying other backend's stats that are excluded +-- from the per-backend stats collection (like the checkpointer). +SELECT count(1) = 0 FROM pg_stat_get_backend_io(:checkpointer_pid); + ?column? +---------- + t +(1 row) + -- Change the tablespace so that the table is rewritten directly, then SELECT -- from it to cause it to be read back into shared buffers. SELECT sum(reads) AS io_sum_shared_before_reads @@ -1521,6 +1576,8 @@ SELECT pg_stat_have_stats('io', 0, 0); SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS io_stats_pre_reset FROM pg_stat_io \gset +SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS my_io_stats_pre_reset + FROM pg_my_stat_io \gset SELECT pg_stat_reset_shared('io'); pg_stat_reset_shared ---------------------- @@ -1535,6 +1592,30 @@ SELECT :io_stats_post_reset < :io_stats_pre_reset; t (1 row) +SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS my_io_stats_post_reset + FROM pg_my_stat_io \gset +-- pg_stat_reset_shared() did not reset backend IO stats +SELECT :my_io_stats_pre_reset <= :my_io_stats_post_reset; + ?column? +---------- + t +(1 row) + +-- but pg_stat_reset_single_backend_io_counters() does +SELECT pg_stat_reset_single_backend_io_counters(pg_backend_pid()); + pg_stat_reset_single_backend_io_counters +------------------------------------------ + +(1 row) + +SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS my_io_stats_post_backend_reset + FROM pg_my_stat_io \gset +SELECT :my_io_stats_pre_reset > :my_io_stats_post_backend_reset; + ?column? +---------- + t +(1 row) + -- test BRIN index doesn't block HOT update CREATE TABLE brin_hot ( id integer PRIMARY KEY, diff --git a/src/test/regress/sql/stats.sql b/src/test/regress/sql/stats.sql index 7147cc2f89..497b2b438a 100644 --- a/src/test/regress/sql/stats.sql +++ b/src/test/regress/sql/stats.sql @@ -595,7 +595,7 @@ SELECT pg_stat_get_replication_slot(NULL); SELECT pg_stat_get_subscription_stats(NULL); --- Test that the following operations are tracked in pg_stat_io: +-- Test that the following operations are tracked in pg_[my_]stat_io: -- - reads of target blocks into shared buffers -- - writes of shared buffers to permanent storage -- - extends of relations using shared buffers @@ -607,20 +607,40 @@ SELECT pg_stat_get_subscription_stats(NULL); -- Create a regular table and insert some data to generate IOCONTEXT_NORMAL -- extends. +SELECT pid AS checkpointer_pid FROM pg_stat_activity + WHERE backend_type = 'checkpointer' \gset SELECT sum(extends) AS io_sum_shared_before_extends FROM pg_stat_io WHERE context = 'normal' AND object = 'relation' \gset +SELECT sum(extends) AS my_io_sum_shared_before_extends + FROM pg_my_stat_io WHERE context = 'normal' AND object = 'relation' \gset +SELECT sum(extends) AS backend_io_sum_shared_before_extends + FROM pg_stat_get_backend_io(pg_backend_pid()) + WHERE context = 'normal' AND object = 'relation' \gset SELECT sum(writes) AS writes, sum(fsyncs) AS fsyncs FROM pg_stat_io WHERE object = 'relation' \gset io_sum_shared_before_ +SELECT sum(writes) AS writes, sum(fsyncs) AS fsyncs + FROM pg_my_stat_io + WHERE object = 'relation' \gset my_io_sum_shared_before_ +SELECT sum(writes) AS writes, sum(fsyncs) AS fsyncs + FROM pg_stat_get_backend_io(pg_backend_pid()) + WHERE object = 'relation' \gset backend_io_sum_shared_before_ CREATE TABLE test_io_shared(a int); INSERT INTO test_io_shared SELECT i FROM generate_series(1,100)i; SELECT pg_stat_force_next_flush(); SELECT sum(extends) AS io_sum_shared_after_extends FROM pg_stat_io WHERE context = 'normal' AND object = 'relation' \gset SELECT :io_sum_shared_after_extends > :io_sum_shared_before_extends; +SELECT sum(extends) AS my_io_sum_shared_after_extends + FROM pg_my_stat_io WHERE context = 'normal' AND object = 'relation' \gset +SELECT :my_io_sum_shared_after_extends > :my_io_sum_shared_before_extends; +SELECT sum(extends) AS backend_io_sum_shared_after_extends + FROM pg_stat_get_backend_io(pg_backend_pid()) + WHERE context = 'normal' AND object = 'relation' \gset +SELECT :backend_io_sum_shared_after_extends > :backend_io_sum_shared_before_extends; -- After a checkpoint, there should be some additional IOCONTEXT_NORMAL writes --- and fsyncs. +-- and fsyncs in the global stats (not for the backend). -- See comment above for rationale for two explicit CHECKPOINTs. CHECKPOINT; CHECKPOINT; @@ -631,6 +651,18 @@ SELECT :io_sum_shared_after_writes > :io_sum_shared_before_writes; SELECT current_setting('fsync') = 'off' OR :io_sum_shared_after_fsyncs > :io_sum_shared_before_fsyncs; +SELECT sum(writes) AS writes, sum(fsyncs) AS fsyncs + FROM pg_my_stat_io + WHERE object = 'relation' \gset my_io_sum_shared_after_ +SELECT :my_io_sum_shared_after_writes >= :my_io_sum_shared_before_writes; +SELECT current_setting('fsync') = 'off' + OR (:my_io_sum_shared_after_fsyncs = :my_io_sum_shared_before_fsyncs + AND :my_io_sum_shared_after_fsyncs= 0); + +-- Don't return any rows if querying other backend's stats that are excluded +-- from the per-backend stats collection (like the checkpointer). +SELECT count(1) = 0 FROM pg_stat_get_backend_io(:checkpointer_pid); + -- Change the tablespace so that the table is rewritten directly, then SELECT -- from it to cause it to be read back into shared buffers. SELECT sum(reads) AS io_sum_shared_before_reads @@ -762,10 +794,21 @@ SELECT :io_sum_bulkwrite_strategy_extends_after > :io_sum_bulkwrite_strategy_ext SELECT pg_stat_have_stats('io', 0, 0); SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS io_stats_pre_reset FROM pg_stat_io \gset +SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS my_io_stats_pre_reset + FROM pg_my_stat_io \gset SELECT pg_stat_reset_shared('io'); SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS io_stats_post_reset FROM pg_stat_io \gset SELECT :io_stats_post_reset < :io_stats_pre_reset; +SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS my_io_stats_post_reset + FROM pg_my_stat_io \gset +-- pg_stat_reset_shared() did not reset backend IO stats +SELECT :my_io_stats_pre_reset <= :my_io_stats_post_reset; +-- but pg_stat_reset_single_backend_io_counters() does +SELECT pg_stat_reset_single_backend_io_counters(pg_backend_pid()); +SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS my_io_stats_post_backend_reset + FROM pg_my_stat_io \gset +SELECT :my_io_stats_pre_reset > :my_io_stats_post_backend_reset; -- test BRIN index doesn't block HOT update diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index b54428b38c..ea676361a5 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2120,6 +2120,7 @@ PgFdwSamplingMethod PgFdwScanState PgIfAddrCallback PgStatShared_Archiver +PgStatShared_Backend PgStatShared_BgWriter PgStatShared_Checkpointer PgStatShared_Common @@ -2135,6 +2136,8 @@ PgStatShared_SLRU PgStatShared_Subscription PgStatShared_Wal PgStat_ArchiverStats +PgStat_Backend +PgStat_BackendPendingIO PgStat_BackendSubEntry PgStat_BgWriterStats PgStat_BktypeIO -- 2.34.1 --msbWIkHyd9NSK3Vl-- ^ permalink raw reply [nested|flat] 25+ messages in thread
* [PATCH v11 2/2] per backend I/O statistics @ 2024-12-18 06:39 Bertrand Drouvot <[email protected]> 0 siblings, 0 replies; 25+ messages in thread From: Bertrand Drouvot @ 2024-12-18 06:39 UTC (permalink / raw) While pg_stat_io provides cluster-wide I/O statistics, this commit adds the ability to track and display per backend I/O statistics. It adds a new statistics kind and 2 new functions: - pg_stat_reset_backend_stats() to be able to reset the stats for a given backend pid. - pg_stat_get_backend_io() to retrieve I/O statistics for a given backend pid. The new KIND is named PGSTAT_KIND_BACKEND as it could be used in the future to store other statistics (than the I/O ones) per backend. The new KIND is a variable-numbered one and has an automatic cap on the maximum number of entries (as its hash key contains the proc number). There is no need to write the per backend I/O stats to disk (no point to see stats for backends that do not exist anymore after a re-start), so using "write_to_file = false". Note that per backend I/O statistics are not collected for the checkpointer, the background writer, the startup process and the autovacuum launcher as those are already visible in pg_stat_io and there is only one of those. XXX: Requires a bump of CATALOG_VERSION_NO. XXX: Requires a bump of PGSTAT_FILE_FORMAT_ID. --- doc/src/sgml/config.sgml | 8 +- doc/src/sgml/monitoring.sgml | 39 ++++ src/backend/catalog/system_functions.sql | 2 + src/backend/utils/activity/Makefile | 1 + src/backend/utils/activity/backend_status.c | 4 + src/backend/utils/activity/meson.build | 1 + src/backend/utils/activity/pgstat.c | 21 +++ src/backend/utils/activity/pgstat_backend.c | 188 +++++++++++++++++++ src/backend/utils/activity/pgstat_io.c | 25 ++- src/backend/utils/activity/pgstat_relation.c | 2 + src/backend/utils/adt/pgstatfuncs.c | 97 +++++++++- src/include/catalog/pg_proc.dat | 13 ++ src/include/pgstat.h | 34 +++- src/include/utils/pgstat_internal.h | 14 ++ src/test/regress/expected/stats.out | 82 +++++++- src/test/regress/sql/stats.sql | 42 ++++- src/tools/pgindent/typedefs.list | 3 + 17 files changed, 549 insertions(+), 27 deletions(-) 10.3% doc/src/sgml/ 34.1% src/backend/utils/activity/ 14.7% src/backend/utils/adt/ 4.7% src/include/catalog/ 8.1% src/include/ 14.6% src/test/regress/expected/ 12.4% src/test/regress/sql/ diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 24bd504c21..fbdd6ce574 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -8403,9 +8403,11 @@ COPY postgres_log FROM '/full/path/to/logfile.csv' WITH csv; displayed in <link linkend="monitoring-pg-stat-database-view"> <structname>pg_stat_database</structname></link>, <link linkend="monitoring-pg-stat-io-view"> - <structname>pg_stat_io</structname></link>, in the output of - <xref linkend="sql-explain"/> when the <literal>BUFFERS</literal> option - is used, in the output of <xref linkend="sql-vacuum"/> when + <structname>pg_stat_io</structname></link>, in the output of the + <link linkend="pg-stat-get-backend-io"> + <function>pg_stat_get_backend_io()</function></link> function, in the + output of <xref linkend="sql-explain"/> when the <literal>BUFFERS</literal> + option is used, in the output of <xref linkend="sql-vacuum"/> when the <literal>VERBOSE</literal> option is used, by autovacuum for auto-vacuums and auto-analyzes, when <xref linkend="guc-log-autovacuum-min-duration"/> is set and by diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml index 840d7f8161..d0d176cc54 100644 --- a/doc/src/sgml/monitoring.sgml +++ b/doc/src/sgml/monitoring.sgml @@ -4790,6 +4790,27 @@ description | Waiting for a newly initialized WAL file to reach durable storage </para></entry> </row> + <row> + <entry id="pg-stat-get-backend-io" role="func_table_entry"><para role="func_signature"> + <indexterm> + <primary>pg_stat_get_backend_io</primary> + </indexterm> + <function>pg_stat_get_backend_io</function> ( <type>integer</type> ) + <returnvalue>setof record</returnvalue> + </para> + <para> + Returns I/O statistics about the backend with the specified + process ID. The output fields are exactly the same as the ones in the + <structname>pg_stat_io</structname> view. + </para> + <para> + The function does not return I/O statistics for the checkpointer, + the background writer, the startup process and the autovacuum launcher + as they are already visible in the <structname>pg_stat_io</structname> + view and there is only one of each. + </para></entry> + </row> + <row> <entry role="func_table_entry"><para role="func_signature"> <indexterm> @@ -4971,6 +4992,24 @@ description | Waiting for a newly initialized WAL file to reach durable storage </para></entry> </row> + <row> + <entry role="func_table_entry"><para role="func_signature"> + <indexterm> + <primary>pg_stat_reset_backend_stats</primary> + </indexterm> + <function>pg_stat_reset_backend_stats</function> ( <type>integer</type> ) + <returnvalue>void</returnvalue> + </para> + <para> + Resets statistics for a single backend with the specified process ID + to zero. + </para> + <para> + This function is restricted to superusers by default, but other users + can be granted EXECUTE to run the function. + </para></entry> + </row> + <row> <entry role="func_table_entry"><para role="func_signature"> <indexterm> diff --git a/src/backend/catalog/system_functions.sql b/src/backend/catalog/system_functions.sql index c51dfca802..14eb99cd47 100644 --- a/src/backend/catalog/system_functions.sql +++ b/src/backend/catalog/system_functions.sql @@ -711,6 +711,8 @@ REVOKE EXECUTE ON FUNCTION pg_stat_reset_single_table_counters(oid) FROM public; REVOKE EXECUTE ON FUNCTION pg_stat_reset_single_function_counters(oid) FROM public; +REVOKE EXECUTE ON FUNCTION pg_stat_reset_backend_stats(integer) FROM public; + REVOKE EXECUTE ON FUNCTION pg_stat_reset_replication_slot(text) FROM public; REVOKE EXECUTE ON FUNCTION pg_stat_have_stats(text, oid, int8) FROM public; diff --git a/src/backend/utils/activity/Makefile b/src/backend/utils/activity/Makefile index b9fd66ea17..24b64a2742 100644 --- a/src/backend/utils/activity/Makefile +++ b/src/backend/utils/activity/Makefile @@ -20,6 +20,7 @@ OBJS = \ backend_status.o \ pgstat.o \ pgstat_archiver.o \ + pgstat_backend.o \ pgstat_bgwriter.o \ pgstat_checkpointer.o \ pgstat_database.o \ diff --git a/src/backend/utils/activity/backend_status.c b/src/backend/utils/activity/backend_status.c index 22c6dc378c..bf33e33a4e 100644 --- a/src/backend/utils/activity/backend_status.c +++ b/src/backend/utils/activity/backend_status.c @@ -426,6 +426,10 @@ pgstat_bestart(void) PGSTAT_END_WRITE_ACTIVITY(vbeentry); + /* Create the backend statistics entry */ + if (pgstat_tracks_backend_bktype(MyBackendType)) + pgstat_create_backend(MyProcNumber); + /* Update app name to current GUC setting */ if (application_name) pgstat_report_appname(application_name); diff --git a/src/backend/utils/activity/meson.build b/src/backend/utils/activity/meson.build index f73c22905c..380d3dd70c 100644 --- a/src/backend/utils/activity/meson.build +++ b/src/backend/utils/activity/meson.build @@ -5,6 +5,7 @@ backend_sources += files( 'backend_status.c', 'pgstat.c', 'pgstat_archiver.c', + 'pgstat_backend.c', 'pgstat_bgwriter.c', 'pgstat_checkpointer.c', 'pgstat_database.c', diff --git a/src/backend/utils/activity/pgstat.c b/src/backend/utils/activity/pgstat.c index b4e357c8a4..b72c779b2c 100644 --- a/src/backend/utils/activity/pgstat.c +++ b/src/backend/utils/activity/pgstat.c @@ -77,6 +77,7 @@ * * Each statistics kind is handled in a dedicated file: * - pgstat_archiver.c + * - pgstat_backend.c * - pgstat_bgwriter.c * - pgstat_checkpointer.c * - pgstat_database.c @@ -358,6 +359,22 @@ static const PgStat_KindInfo pgstat_kind_builtin_infos[PGSTAT_KIND_BUILTIN_SIZE] .reset_timestamp_cb = pgstat_subscription_reset_timestamp_cb, }, + [PGSTAT_KIND_BACKEND] = { + .name = "backend", + + .fixed_amount = false, + .write_to_file = false, + + .accessed_across_databases = true, + + .shared_size = sizeof(PgStatShared_Backend), + .shared_data_off = offsetof(PgStatShared_Backend, stats), + .shared_data_len = sizeof(((PgStatShared_Backend *) 0)->stats), + .pending_size = sizeof(PgStat_BackendPendingIO), + + .flush_pending_cb = pgstat_backend_flush_cb, + .reset_timestamp_cb = pgstat_backend_reset_timestamp_cb, + }, /* stats for fixed-numbered (mostly 1) objects */ @@ -602,6 +619,10 @@ pgstat_shutdown_hook(int code, Datum arg) Assert(dlist_is_empty(&pgStatPending)); dlist_init(&pgStatPending); + /* drop the backend stats entry */ + if (!pgstat_drop_entry(PGSTAT_KIND_BACKEND, InvalidOid, MyProcNumber)) + pgstat_request_entry_refs_gc(); + pgstat_detach_shmem(); #ifdef USE_ASSERT_CHECKING diff --git a/src/backend/utils/activity/pgstat_backend.c b/src/backend/utils/activity/pgstat_backend.c new file mode 100644 index 0000000000..6b2c9baa8c --- /dev/null +++ b/src/backend/utils/activity/pgstat_backend.c @@ -0,0 +1,188 @@ +/* ------------------------------------------------------------------------- + * + * pgstat_backend.c + * Implementation of backend statistics. + * + * This file contains the implementation of backend statistics. It is kept + * separate from pgstat.c to enforce the line between the statistics access / + * storage implementation and the details about individual types of + * statistics. + * + * This statistics kind uses a proc number as object ID for the hash table + * of pgstats. Entries are created each time a process is spawned, and are + * dropped when the process exits. These are not written to the pgstats file + * on disk. + * + * Copyright (c) 2001-2024, PostgreSQL Global Development Group + * + * IDENTIFICATION + * src/backend/utils/activity/pgstat_backend.c + * ------------------------------------------------------------------------- + */ + +#include "postgres.h" + +#include "utils/pgstat_internal.h" + +/* + * Returns statistics of a backend by proc number. + */ +PgStat_Backend * +pgstat_fetch_stat_backend(ProcNumber procNumber) +{ + PgStat_Backend *backend_entry; + + backend_entry = (PgStat_Backend *) pgstat_fetch_entry(PGSTAT_KIND_BACKEND, + InvalidOid, procNumber); + + return backend_entry; +} + +/* + * Flush out locally pending backend statistics + * + * If no stats have been recorded, this function returns false. + */ +bool +pgstat_backend_flush_cb(PgStat_EntryRef *entry_ref, bool nowait) +{ + PgStatShared_Backend *shbackendioent; + PgStat_BackendPendingIO *pendingent; + PgStat_BktypeIO *bktype_shstats; + + if (!pgstat_lock_entry(entry_ref, nowait)) + return false; + + shbackendioent = (PgStatShared_Backend *) entry_ref->shared_stats; + bktype_shstats = &shbackendioent->stats.stats; + pendingent = (PgStat_BackendPendingIO *) entry_ref->pending; + + for (int io_object = 0; io_object < IOOBJECT_NUM_TYPES; io_object++) + { + for (int io_context = 0; io_context < IOCONTEXT_NUM_TYPES; io_context++) + { + for (int io_op = 0; io_op < IOOP_NUM_TYPES; io_op++) + { + instr_time time; + + bktype_shstats->counts[io_object][io_context][io_op] += + pendingent->counts[io_object][io_context][io_op]; + + time = pendingent->pending_times[io_object][io_context][io_op]; + + bktype_shstats->times[io_object][io_context][io_op] += + INSTR_TIME_GET_MICROSEC(time); + } + } + } + + pgstat_unlock_entry(entry_ref); + + return true; +} + +/* + * Simpler wrapper of pgstat_backend_flush_cb() + */ +void +pgstat_flush_backend(bool nowait) +{ + PgStat_EntryRef *entry_ref; + + if (!pgstat_tracks_backend_bktype(MyBackendType)) + return; + + entry_ref = pgstat_get_entry_ref(PGSTAT_KIND_BACKEND, InvalidOid, + MyProcNumber, false, NULL); + (void) pgstat_backend_flush_cb(entry_ref, nowait); +} + +/* + * Create backend statistics entry for proc number. + */ +void +pgstat_create_backend(ProcNumber procnum) +{ + PgStat_EntryRef *entry_ref; + PgStatShared_Backend *shstatent; + + entry_ref = pgstat_prep_pending_entry(PGSTAT_KIND_BACKEND, InvalidOid, + procnum, NULL); + + shstatent = (PgStatShared_Backend *) entry_ref->shared_stats; + + /* + * NB: need to accept that there might be stats from an older backend, + * e.g. if we previously used this proc number. + */ + memset(&shstatent->stats, 0, sizeof(shstatent->stats)); +} + +/* + * Find or create a local PgStat_BackendPendingIO entry for proc number. + */ +PgStat_BackendPendingIO * +pgstat_prep_backend_pending(ProcNumber procnum) +{ + PgStat_EntryRef *entry_ref; + + entry_ref = pgstat_prep_pending_entry(PGSTAT_KIND_BACKEND, InvalidOid, + procnum, NULL); + + return entry_ref->pending; +} + +/* + * Backend statistics are not collected for all BackendTypes. + * + * The following BackendTypes do not participate in the backend stats + * subsystem: + * - The same and for the same reasons as in pgstat_tracks_io_bktype(). + * - B_BG_WRITER, B_CHECKPOINTER, B_STARTUP and B_AUTOVAC_LAUNCHER because their + * I/O stats are already visible in pg_stat_io and there is only one of those. + * + * Function returns true if BackendType participates in the backend stats + * subsystem and false if it does not. + * + * When adding a new BackendType, also consider adding relevant restrictions to + * pgstat_tracks_io_object() and pgstat_tracks_io_op(). + */ +bool +pgstat_tracks_backend_bktype(BackendType bktype) +{ + /* + * List every type so that new backend types trigger a warning about + * needing to adjust this switch. + */ + switch (bktype) + { + case B_INVALID: + case B_AUTOVAC_LAUNCHER: + case B_DEAD_END_BACKEND: + case B_ARCHIVER: + case B_LOGGER: + case B_WAL_RECEIVER: + case B_WAL_WRITER: + case B_WAL_SUMMARIZER: + case B_BG_WRITER: + case B_CHECKPOINTER: + case B_STARTUP: + return false; + + case B_AUTOVAC_WORKER: + case B_BACKEND: + case B_BG_WORKER: + case B_STANDALONE_BACKEND: + case B_SLOTSYNC_WORKER: + case B_WAL_SENDER: + return true; + } + + return false; +} + +void +pgstat_backend_reset_timestamp_cb(PgStatShared_Common *header, TimestampTz ts) +{ + ((PgStatShared_Backend *) header)->stats.stat_reset_timestamp = ts; +} diff --git a/src/backend/utils/activity/pgstat_io.c b/src/backend/utils/activity/pgstat_io.c index f9883af2b3..011a3326da 100644 --- a/src/backend/utils/activity/pgstat_io.c +++ b/src/backend/utils/activity/pgstat_io.c @@ -20,14 +20,6 @@ #include "storage/bufmgr.h" #include "utils/pgstat_internal.h" - -typedef struct PgStat_PendingIO -{ - PgStat_Counter counts[IOOBJECT_NUM_TYPES][IOCONTEXT_NUM_TYPES][IOOP_NUM_TYPES]; - instr_time pending_times[IOOBJECT_NUM_TYPES][IOCONTEXT_NUM_TYPES][IOOP_NUM_TYPES]; -} PgStat_PendingIO; - - static PgStat_PendingIO PendingIOStats; static bool have_iostats = false; @@ -87,6 +79,14 @@ pgstat_count_io_op_n(IOObject io_object, IOContext io_context, IOOp io_op, uint3 Assert((unsigned int) io_op < IOOP_NUM_TYPES); Assert(pgstat_tracks_io_op(MyBackendType, io_object, io_context, io_op)); + if (pgstat_tracks_backend_bktype(MyBackendType)) + { + PgStat_PendingIO *entry_ref; + + entry_ref = pgstat_prep_backend_pending(MyProcNumber); + entry_ref->counts[io_object][io_context][io_op] += cnt; + } + PendingIOStats.counts[io_object][io_context][io_op] += cnt; have_iostats = true; @@ -148,6 +148,15 @@ pgstat_count_io_op_time(IOObject io_object, IOContext io_context, IOOp io_op, INSTR_TIME_ADD(PendingIOStats.pending_times[io_object][io_context][io_op], io_time); + + if (pgstat_tracks_backend_bktype(MyBackendType)) + { + PgStat_PendingIO *entry_ref; + + entry_ref = pgstat_prep_backend_pending(MyProcNumber); + INSTR_TIME_ADD(entry_ref->pending_times[io_object][io_context][io_op], + io_time); + } } pgstat_count_io_op_n(io_object, io_context, io_op, cnt); diff --git a/src/backend/utils/activity/pgstat_relation.c b/src/backend/utils/activity/pgstat_relation.c index faba8b64d2..85e65557bb 100644 --- a/src/backend/utils/activity/pgstat_relation.c +++ b/src/backend/utils/activity/pgstat_relation.c @@ -264,6 +264,7 @@ pgstat_report_vacuum(Oid tableoid, bool shared, * VACUUM command has processed all tables and committed. */ pgstat_flush_io(false); + pgstat_flush_backend(false); } /* @@ -350,6 +351,7 @@ pgstat_report_analyze(Relation rel, /* see pgstat_report_vacuum() */ pgstat_flush_io(false); + pgstat_flush_backend(false); } /* diff --git a/src/backend/utils/adt/pgstatfuncs.c b/src/backend/utils/adt/pgstatfuncs.c index efd35f700a..8387e11a93 100644 --- a/src/backend/utils/adt/pgstatfuncs.c +++ b/src/backend/utils/adt/pgstatfuncs.c @@ -1274,8 +1274,9 @@ pg_stat_get_buf_alloc(PG_FUNCTION_ARGS) } /* -* When adding a new column to the pg_stat_io view, add a new enum value -* here above IO_NUM_COLUMNS. +* When adding a new column to the pg_stat_io view and the +* pg_stat_get_backend_io() function, add a new enum value here above +* IO_NUM_COLUMNS. */ typedef enum io_stat_col { @@ -1368,9 +1369,9 @@ pg_stat_us_to_ms(PgStat_Counter val_ms) /* * pg_stat_io_build_tuples * - * Helper routine for pg_stat_get_io() filling in a result tuplestore with one - * tuple for each object and each context supported by the caller, based on the - * contents of bktype_stats. + * Helper routine for pg_stat_get_io() and pg_stat_get_backend_io(), + * filling in a result tuplestore with one tuple for each object and each + * context supported by the caller, based on the contents of bktype_stats. */ static void pg_stat_io_build_tuples(ReturnSetInfo *rsinfo, @@ -1494,6 +1495,68 @@ pg_stat_get_io(PG_FUNCTION_ARGS) return (Datum) 0; } +/* + * Returns I/O statistics for a backend with given PID. + */ +Datum +pg_stat_get_backend_io(PG_FUNCTION_ARGS) +{ + ReturnSetInfo *rsinfo; + BackendType bktype; + int pid; + PGPROC *proc; + ProcNumber procNumber; + PgStat_Backend *backend_stats; + PgStat_BktypeIO *bktype_stats; + PgBackendStatus *beentry; + + InitMaterializedSRF(fcinfo, 0); + rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; + + pid = PG_GETARG_INT32(0); + proc = BackendPidGetProc(pid); + + /* + * This could be an auxiliary process but these do not report backend + * statistics due to pgstat_tracks_backend_bktype(), so there is no need + * for an extra call to AuxiliaryPidGetProc(). + */ + if (!proc) + return (Datum) 0; + + procNumber = GetNumberFromPGProc(proc); + + backend_stats = pgstat_fetch_stat_backend(procNumber); + + if (!backend_stats) + return (Datum) 0; + + beentry = pgstat_get_beentry_by_proc_number(procNumber); + bktype = beentry->st_backendType; + + /* if PID does not match, leave */ + if (beentry->st_procpid != pid) + return (Datum) 0; + + /* backend may be gone, so recheck in case */ + if (bktype == B_INVALID) + return (Datum) 0; + + bktype_stats = &backend_stats->stats; + + /* + * In Assert builds, we can afford an extra loop through all of the + * counters (in pg_stat_io_build_tuples()), checking that only expected + * stats are non-zero, since it keeps the non-Assert code cleaner. + */ + Assert(pgstat_bktype_io_stats_valid(bktype_stats, bktype)); + + /* save tuples with data from this PgStat_BktypeIO */ + pg_stat_io_build_tuples(rsinfo, bktype_stats, bktype, + backend_stats->stat_reset_timestamp); + return (Datum) 0; +} + /* * Returns statistics of WAL activity */ @@ -1799,6 +1862,30 @@ pg_stat_reset_single_function_counters(PG_FUNCTION_ARGS) PG_RETURN_VOID(); } +/* + * Reset statistics of backend with given PID. + */ +Datum +pg_stat_reset_backend_stats(PG_FUNCTION_ARGS) +{ + PGPROC *proc; + int backend_pid = PG_GETARG_INT32(0); + + proc = BackendPidGetProc(backend_pid); + + /* + * This could be an auxiliary process but these do not report backend + * statistics due to pgstat_tracks_backend_bktype(), so there is no need + * for an extra call to AuxiliaryPidGetProc(). + */ + if (!proc) + PG_RETURN_VOID(); + + pgstat_reset(PGSTAT_KIND_BACKEND, InvalidOid, GetNumberFromPGProc(proc)); + + PG_RETURN_VOID(); +} + /* Reset SLRU counters (a specific one or all of them). */ Datum pg_stat_reset_slru(PG_FUNCTION_ARGS) diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index 0f22c21723..2dcc2d42da 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -5913,6 +5913,15 @@ proargnames => '{backend_type,object,context,reads,read_time,writes,write_time,writebacks,writeback_time,extends,extend_time,op_bytes,hits,evictions,reuses,fsyncs,fsync_time,stats_reset}', prosrc => 'pg_stat_get_io' }, +{ oid => '8806', descr => 'statistics: backend IO statistics', + proname => 'pg_stat_get_backend_io', prorows => '5', proretset => 't', + provolatile => 'v', proparallel => 'r', prorettype => 'record', + proargtypes => 'int4', + proallargtypes => '{int4,text,text,text,int8,float8,int8,float8,int8,float8,int8,float8,int8,int8,int8,int8,int8,float8,timestamptz}', + proargmodes => '{i,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o}', + proargnames => '{backend_pid,backend_type,object,context,reads,read_time,writes,write_time,writebacks,writeback_time,extends,extend_time,op_bytes,hits,evictions,reuses,fsyncs,fsync_time,stats_reset}', + prosrc => 'pg_stat_get_backend_io' }, + { oid => '1136', descr => 'statistics: information about WAL activity', proname => 'pg_stat_get_wal', proisstrict => 'f', provolatile => 's', proparallel => 'r', prorettype => 'record', proargtypes => '', @@ -6052,6 +6061,10 @@ proname => 'pg_stat_reset_single_function_counters', provolatile => 'v', prorettype => 'void', proargtypes => 'oid', prosrc => 'pg_stat_reset_single_function_counters' }, +{ oid => '8807', descr => 'statistics: reset statistics for a single backend', + proname => 'pg_stat_reset_backend_stats', provolatile => 'v', + prorettype => 'void', proargtypes => 'int4', + prosrc => 'pg_stat_reset_backend_stats' }, { oid => '2307', descr => 'statistics: reset collected statistics for a single SLRU', proname => 'pg_stat_reset_slru', proisstrict => 'f', provolatile => 'v', diff --git a/src/include/pgstat.h b/src/include/pgstat.h index ebfeef2f46..56ed1b8bb3 100644 --- a/src/include/pgstat.h +++ b/src/include/pgstat.h @@ -49,14 +49,15 @@ #define PGSTAT_KIND_FUNCTION 3 /* per-function statistics */ #define PGSTAT_KIND_REPLSLOT 4 /* per-slot statistics */ #define PGSTAT_KIND_SUBSCRIPTION 5 /* per-subscription statistics */ +#define PGSTAT_KIND_BACKEND 6 /* per-backend statistics */ /* stats for fixed-numbered objects */ -#define PGSTAT_KIND_ARCHIVER 6 -#define PGSTAT_KIND_BGWRITER 7 -#define PGSTAT_KIND_CHECKPOINTER 8 -#define PGSTAT_KIND_IO 9 -#define PGSTAT_KIND_SLRU 10 -#define PGSTAT_KIND_WAL 11 +#define PGSTAT_KIND_ARCHIVER 7 +#define PGSTAT_KIND_BGWRITER 8 +#define PGSTAT_KIND_CHECKPOINTER 9 +#define PGSTAT_KIND_IO 10 +#define PGSTAT_KIND_SLRU 11 +#define PGSTAT_KIND_WAL 12 #define PGSTAT_KIND_BUILTIN_MIN PGSTAT_KIND_DATABASE #define PGSTAT_KIND_BUILTIN_MAX PGSTAT_KIND_WAL @@ -362,12 +363,26 @@ typedef struct PgStat_BktypeIO PgStat_Counter times[IOOBJECT_NUM_TYPES][IOCONTEXT_NUM_TYPES][IOOP_NUM_TYPES]; } PgStat_BktypeIO; +typedef struct PgStat_PendingIO +{ + PgStat_Counter counts[IOOBJECT_NUM_TYPES][IOCONTEXT_NUM_TYPES][IOOP_NUM_TYPES]; + instr_time pending_times[IOOBJECT_NUM_TYPES][IOCONTEXT_NUM_TYPES][IOOP_NUM_TYPES]; +} PgStat_PendingIO; + typedef struct PgStat_IO { TimestampTz stat_reset_timestamp; PgStat_BktypeIO stats[BACKEND_NUM_TYPES]; } PgStat_IO; +/* Backend statistics store the same amount of IO data as PGSTAT_KIND_IO */ +typedef PgStat_PendingIO PgStat_BackendPendingIO; + +typedef struct PgStat_Backend +{ + TimestampTz stat_reset_timestamp; + PgStat_BktypeIO stats; +} PgStat_Backend; typedef struct PgStat_StatDBEntry { @@ -549,6 +564,13 @@ extern bool pgstat_have_entry(PgStat_Kind kind, Oid dboid, uint64 objid); extern void pgstat_report_archiver(const char *xlog, bool failed); extern PgStat_ArchiverStats *pgstat_fetch_stat_archiver(void); +/* + * Functions in pgstat_backend.c + */ + +extern PgStat_Backend *pgstat_fetch_stat_backend(ProcNumber procNumber); +extern bool pgstat_tracks_backend_bktype(BackendType bktype); +extern void pgstat_create_backend(ProcNumber procnum); /* * Functions in pgstat_bgwriter.c diff --git a/src/include/utils/pgstat_internal.h b/src/include/utils/pgstat_internal.h index 7338bc1e28..811ed9b005 100644 --- a/src/include/utils/pgstat_internal.h +++ b/src/include/utils/pgstat_internal.h @@ -450,6 +450,11 @@ typedef struct PgStatShared_ReplSlot PgStat_StatReplSlotEntry stats; } PgStatShared_ReplSlot; +typedef struct PgStatShared_Backend +{ + PgStatShared_Common header; + PgStat_Backend stats; +} PgStatShared_Backend; /* * Central shared memory entry for the cumulative stats system. @@ -604,6 +609,15 @@ extern void pgstat_archiver_init_shmem_cb(void *stats); extern void pgstat_archiver_reset_all_cb(TimestampTz ts); extern void pgstat_archiver_snapshot_cb(void); +/* + * Functions in pgstat_backend.c + */ + +extern void pgstat_flush_backend(bool nowait); + +extern PgStat_BackendPendingIO *pgstat_prep_backend_pending(ProcNumber procnum); +extern bool pgstat_backend_flush_cb(PgStat_EntryRef *entry_ref, bool nowait); +extern void pgstat_backend_reset_timestamp_cb(PgStatShared_Common *header, TimestampTz ts); /* * Functions in pgstat_bgwriter.c diff --git a/src/test/regress/expected/stats.out b/src/test/regress/expected/stats.out index 56771f83ed..150b6dcf74 100644 --- a/src/test/regress/expected/stats.out +++ b/src/test/regress/expected/stats.out @@ -1249,7 +1249,8 @@ SELECT pg_stat_get_subscription_stats(NULL); (1 row) --- Test that the following operations are tracked in pg_stat_io: +-- Test that the following operations are tracked in pg_stat_io and in +-- backend stats: -- - reads of target blocks into shared buffers -- - writes of shared buffers to permanent storage -- - extends of relations using shared buffers @@ -1259,11 +1260,19 @@ SELECT pg_stat_get_subscription_stats(NULL); -- be sure of the state of shared buffers at the point the test is run. -- Create a regular table and insert some data to generate IOCONTEXT_NORMAL -- extends. +SELECT pid AS checkpointer_pid FROM pg_stat_activity + WHERE backend_type = 'checkpointer' \gset SELECT sum(extends) AS io_sum_shared_before_extends FROM pg_stat_io WHERE context = 'normal' AND object = 'relation' \gset +SELECT sum(extends) AS my_io_sum_shared_before_extends + FROM pg_stat_get_backend_io(pg_backend_pid()) + WHERE context = 'normal' AND object = 'relation' \gset SELECT sum(writes) AS writes, sum(fsyncs) AS fsyncs FROM pg_stat_io WHERE object = 'relation' \gset io_sum_shared_before_ +SELECT sum(writes) AS writes, sum(fsyncs) AS fsyncs + FROM pg_stat_get_backend_io(pg_backend_pid()) + WHERE object = 'relation' \gset my_io_sum_shared_before_ CREATE TABLE test_io_shared(a int); INSERT INTO test_io_shared SELECT i FROM generate_series(1,100)i; SELECT pg_stat_force_next_flush(); @@ -1280,8 +1289,17 @@ SELECT :io_sum_shared_after_extends > :io_sum_shared_before_extends; t (1 row) +SELECT sum(extends) AS my_io_sum_shared_after_extends + FROM pg_stat_get_backend_io(pg_backend_pid()) + WHERE context = 'normal' AND object = 'relation' \gset +SELECT :my_io_sum_shared_after_extends > :my_io_sum_shared_before_extends; + ?column? +---------- + t +(1 row) + -- After a checkpoint, there should be some additional IOCONTEXT_NORMAL writes --- and fsyncs. +-- and fsyncs in the global stats (not for the backend). -- See comment above for rationale for two explicit CHECKPOINTs. CHECKPOINT; CHECKPOINT; @@ -1301,6 +1319,23 @@ SELECT current_setting('fsync') = 'off' t (1 row) +SELECT sum(writes) AS writes, sum(fsyncs) AS fsyncs + FROM pg_stat_get_backend_io(pg_backend_pid()) + WHERE object = 'relation' \gset my_io_sum_shared_after_ +SELECT :my_io_sum_shared_after_writes >= :my_io_sum_shared_before_writes; + ?column? +---------- + t +(1 row) + +SELECT current_setting('fsync') = 'off' + OR (:my_io_sum_shared_after_fsyncs = :my_io_sum_shared_before_fsyncs + AND :my_io_sum_shared_after_fsyncs= 0); + ?column? +---------- + t +(1 row) + -- Change the tablespace so that the table is rewritten directly, then SELECT -- from it to cause it to be read back into shared buffers. SELECT sum(reads) AS io_sum_shared_before_reads @@ -1521,6 +1556,8 @@ SELECT pg_stat_have_stats('io', 0, 0); SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS io_stats_pre_reset FROM pg_stat_io \gset +SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS my_io_stats_pre_reset + FROM pg_stat_get_backend_io(pg_backend_pid()) \gset SELECT pg_stat_reset_shared('io'); pg_stat_reset_shared ---------------------- @@ -1535,6 +1572,47 @@ SELECT :io_stats_post_reset < :io_stats_pre_reset; t (1 row) +SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS my_io_stats_post_reset + FROM pg_stat_get_backend_io(pg_backend_pid()) \gset +-- pg_stat_reset_shared() did not reset backend IO stats +SELECT :my_io_stats_pre_reset <= :my_io_stats_post_reset; + ?column? +---------- + t +(1 row) + +-- but pg_stat_reset_backend_stats() does +SELECT pg_stat_reset_backend_stats(pg_backend_pid()); + pg_stat_reset_backend_stats +----------------------------- + +(1 row) + +SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS my_io_stats_post_backend_reset + FROM pg_stat_get_backend_io(pg_backend_pid()) \gset +SELECT :my_io_stats_pre_reset > :my_io_stats_post_backend_reset; + ?column? +---------- + t +(1 row) + +-- Check invalid input for pg_stat_get_backend_io() +SELECT pg_stat_get_backend_io(NULL); + pg_stat_get_backend_io +------------------------ +(0 rows) + +SELECT pg_stat_get_backend_io(0); + pg_stat_get_backend_io +------------------------ +(0 rows) + +-- Auxiliary processes return no data. +SELECT pg_stat_get_backend_io(:checkpointer_pid); + pg_stat_get_backend_io +------------------------ +(0 rows) + -- test BRIN index doesn't block HOT update CREATE TABLE brin_hot ( id integer PRIMARY KEY, diff --git a/src/test/regress/sql/stats.sql b/src/test/regress/sql/stats.sql index 7147cc2f89..1e7d0ff665 100644 --- a/src/test/regress/sql/stats.sql +++ b/src/test/regress/sql/stats.sql @@ -595,7 +595,8 @@ SELECT pg_stat_get_replication_slot(NULL); SELECT pg_stat_get_subscription_stats(NULL); --- Test that the following operations are tracked in pg_stat_io: +-- Test that the following operations are tracked in pg_stat_io and in +-- backend stats: -- - reads of target blocks into shared buffers -- - writes of shared buffers to permanent storage -- - extends of relations using shared buffers @@ -607,20 +608,32 @@ SELECT pg_stat_get_subscription_stats(NULL); -- Create a regular table and insert some data to generate IOCONTEXT_NORMAL -- extends. +SELECT pid AS checkpointer_pid FROM pg_stat_activity + WHERE backend_type = 'checkpointer' \gset SELECT sum(extends) AS io_sum_shared_before_extends FROM pg_stat_io WHERE context = 'normal' AND object = 'relation' \gset +SELECT sum(extends) AS my_io_sum_shared_before_extends + FROM pg_stat_get_backend_io(pg_backend_pid()) + WHERE context = 'normal' AND object = 'relation' \gset SELECT sum(writes) AS writes, sum(fsyncs) AS fsyncs FROM pg_stat_io WHERE object = 'relation' \gset io_sum_shared_before_ +SELECT sum(writes) AS writes, sum(fsyncs) AS fsyncs + FROM pg_stat_get_backend_io(pg_backend_pid()) + WHERE object = 'relation' \gset my_io_sum_shared_before_ CREATE TABLE test_io_shared(a int); INSERT INTO test_io_shared SELECT i FROM generate_series(1,100)i; SELECT pg_stat_force_next_flush(); SELECT sum(extends) AS io_sum_shared_after_extends FROM pg_stat_io WHERE context = 'normal' AND object = 'relation' \gset SELECT :io_sum_shared_after_extends > :io_sum_shared_before_extends; +SELECT sum(extends) AS my_io_sum_shared_after_extends + FROM pg_stat_get_backend_io(pg_backend_pid()) + WHERE context = 'normal' AND object = 'relation' \gset +SELECT :my_io_sum_shared_after_extends > :my_io_sum_shared_before_extends; -- After a checkpoint, there should be some additional IOCONTEXT_NORMAL writes --- and fsyncs. +-- and fsyncs in the global stats (not for the backend). -- See comment above for rationale for two explicit CHECKPOINTs. CHECKPOINT; CHECKPOINT; @@ -630,6 +643,13 @@ SELECT sum(writes) AS writes, sum(fsyncs) AS fsyncs SELECT :io_sum_shared_after_writes > :io_sum_shared_before_writes; SELECT current_setting('fsync') = 'off' OR :io_sum_shared_after_fsyncs > :io_sum_shared_before_fsyncs; +SELECT sum(writes) AS writes, sum(fsyncs) AS fsyncs + FROM pg_stat_get_backend_io(pg_backend_pid()) + WHERE object = 'relation' \gset my_io_sum_shared_after_ +SELECT :my_io_sum_shared_after_writes >= :my_io_sum_shared_before_writes; +SELECT current_setting('fsync') = 'off' + OR (:my_io_sum_shared_after_fsyncs = :my_io_sum_shared_before_fsyncs + AND :my_io_sum_shared_after_fsyncs= 0); -- Change the tablespace so that the table is rewritten directly, then SELECT -- from it to cause it to be read back into shared buffers. @@ -762,11 +782,27 @@ SELECT :io_sum_bulkwrite_strategy_extends_after > :io_sum_bulkwrite_strategy_ext SELECT pg_stat_have_stats('io', 0, 0); SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS io_stats_pre_reset FROM pg_stat_io \gset +SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS my_io_stats_pre_reset + FROM pg_stat_get_backend_io(pg_backend_pid()) \gset SELECT pg_stat_reset_shared('io'); SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS io_stats_post_reset FROM pg_stat_io \gset SELECT :io_stats_post_reset < :io_stats_pre_reset; - +SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS my_io_stats_post_reset + FROM pg_stat_get_backend_io(pg_backend_pid()) \gset +-- pg_stat_reset_shared() did not reset backend IO stats +SELECT :my_io_stats_pre_reset <= :my_io_stats_post_reset; +-- but pg_stat_reset_backend_stats() does +SELECT pg_stat_reset_backend_stats(pg_backend_pid()); +SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS my_io_stats_post_backend_reset + FROM pg_stat_get_backend_io(pg_backend_pid()) \gset +SELECT :my_io_stats_pre_reset > :my_io_stats_post_backend_reset; + +-- Check invalid input for pg_stat_get_backend_io() +SELECT pg_stat_get_backend_io(NULL); +SELECT pg_stat_get_backend_io(0); +-- Auxiliary processes return no data. +SELECT pg_stat_get_backend_io(:checkpointer_pid); -- test BRIN index doesn't block HOT update CREATE TABLE brin_hot ( diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index ce33e55bf1..398dd92527 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2121,6 +2121,7 @@ PgFdwSamplingMethod PgFdwScanState PgIfAddrCallback PgStatShared_Archiver +PgStatShared_Backend PgStatShared_BgWriter PgStatShared_Checkpointer PgStatShared_Common @@ -2136,6 +2137,8 @@ PgStatShared_SLRU PgStatShared_Subscription PgStatShared_Wal PgStat_ArchiverStats +PgStat_Backend +PgStat_BackendPendingIO PgStat_BackendSubEntry PgStat_BgWriterStats PgStat_BktypeIO -- 2.34.1 --YrW53rPX+W0tiwfU-- ^ permalink raw reply [nested|flat] 25+ messages in thread
* [PATCH v11 2/2] per backend I/O statistics @ 2024-12-18 06:39 Bertrand Drouvot <[email protected]> 0 siblings, 0 replies; 25+ messages in thread From: Bertrand Drouvot @ 2024-12-18 06:39 UTC (permalink / raw) While pg_stat_io provides cluster-wide I/O statistics, this commit adds the ability to track and display per backend I/O statistics. It adds a new statistics kind and 2 new functions: - pg_stat_reset_backend_stats() to be able to reset the stats for a given backend pid. - pg_stat_get_backend_io() to retrieve I/O statistics for a given backend pid. The new KIND is named PGSTAT_KIND_BACKEND as it could be used in the future to store other statistics (than the I/O ones) per backend. The new KIND is a variable-numbered one and has an automatic cap on the maximum number of entries (as its hash key contains the proc number). There is no need to write the per backend I/O stats to disk (no point to see stats for backends that do not exist anymore after a re-start), so using "write_to_file = false". Note that per backend I/O statistics are not collected for the checkpointer, the background writer, the startup process and the autovacuum launcher as those are already visible in pg_stat_io and there is only one of those. XXX: Requires a bump of CATALOG_VERSION_NO. XXX: Requires a bump of PGSTAT_FILE_FORMAT_ID. --- doc/src/sgml/config.sgml | 8 +- doc/src/sgml/monitoring.sgml | 39 ++++ src/backend/catalog/system_functions.sql | 2 + src/backend/utils/activity/Makefile | 1 + src/backend/utils/activity/backend_status.c | 4 + src/backend/utils/activity/meson.build | 1 + src/backend/utils/activity/pgstat.c | 21 +++ src/backend/utils/activity/pgstat_backend.c | 188 +++++++++++++++++++ src/backend/utils/activity/pgstat_io.c | 25 ++- src/backend/utils/activity/pgstat_relation.c | 2 + src/backend/utils/adt/pgstatfuncs.c | 97 +++++++++- src/include/catalog/pg_proc.dat | 13 ++ src/include/pgstat.h | 34 +++- src/include/utils/pgstat_internal.h | 14 ++ src/test/regress/expected/stats.out | 82 +++++++- src/test/regress/sql/stats.sql | 42 ++++- src/tools/pgindent/typedefs.list | 3 + 17 files changed, 549 insertions(+), 27 deletions(-) 10.3% doc/src/sgml/ 34.1% src/backend/utils/activity/ 14.7% src/backend/utils/adt/ 4.7% src/include/catalog/ 8.1% src/include/ 14.6% src/test/regress/expected/ 12.4% src/test/regress/sql/ diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 24bd504c21..fbdd6ce574 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -8403,9 +8403,11 @@ COPY postgres_log FROM '/full/path/to/logfile.csv' WITH csv; displayed in <link linkend="monitoring-pg-stat-database-view"> <structname>pg_stat_database</structname></link>, <link linkend="monitoring-pg-stat-io-view"> - <structname>pg_stat_io</structname></link>, in the output of - <xref linkend="sql-explain"/> when the <literal>BUFFERS</literal> option - is used, in the output of <xref linkend="sql-vacuum"/> when + <structname>pg_stat_io</structname></link>, in the output of the + <link linkend="pg-stat-get-backend-io"> + <function>pg_stat_get_backend_io()</function></link> function, in the + output of <xref linkend="sql-explain"/> when the <literal>BUFFERS</literal> + option is used, in the output of <xref linkend="sql-vacuum"/> when the <literal>VERBOSE</literal> option is used, by autovacuum for auto-vacuums and auto-analyzes, when <xref linkend="guc-log-autovacuum-min-duration"/> is set and by diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml index 840d7f8161..d0d176cc54 100644 --- a/doc/src/sgml/monitoring.sgml +++ b/doc/src/sgml/monitoring.sgml @@ -4790,6 +4790,27 @@ description | Waiting for a newly initialized WAL file to reach durable storage </para></entry> </row> + <row> + <entry id="pg-stat-get-backend-io" role="func_table_entry"><para role="func_signature"> + <indexterm> + <primary>pg_stat_get_backend_io</primary> + </indexterm> + <function>pg_stat_get_backend_io</function> ( <type>integer</type> ) + <returnvalue>setof record</returnvalue> + </para> + <para> + Returns I/O statistics about the backend with the specified + process ID. The output fields are exactly the same as the ones in the + <structname>pg_stat_io</structname> view. + </para> + <para> + The function does not return I/O statistics for the checkpointer, + the background writer, the startup process and the autovacuum launcher + as they are already visible in the <structname>pg_stat_io</structname> + view and there is only one of each. + </para></entry> + </row> + <row> <entry role="func_table_entry"><para role="func_signature"> <indexterm> @@ -4971,6 +4992,24 @@ description | Waiting for a newly initialized WAL file to reach durable storage </para></entry> </row> + <row> + <entry role="func_table_entry"><para role="func_signature"> + <indexterm> + <primary>pg_stat_reset_backend_stats</primary> + </indexterm> + <function>pg_stat_reset_backend_stats</function> ( <type>integer</type> ) + <returnvalue>void</returnvalue> + </para> + <para> + Resets statistics for a single backend with the specified process ID + to zero. + </para> + <para> + This function is restricted to superusers by default, but other users + can be granted EXECUTE to run the function. + </para></entry> + </row> + <row> <entry role="func_table_entry"><para role="func_signature"> <indexterm> diff --git a/src/backend/catalog/system_functions.sql b/src/backend/catalog/system_functions.sql index c51dfca802..14eb99cd47 100644 --- a/src/backend/catalog/system_functions.sql +++ b/src/backend/catalog/system_functions.sql @@ -711,6 +711,8 @@ REVOKE EXECUTE ON FUNCTION pg_stat_reset_single_table_counters(oid) FROM public; REVOKE EXECUTE ON FUNCTION pg_stat_reset_single_function_counters(oid) FROM public; +REVOKE EXECUTE ON FUNCTION pg_stat_reset_backend_stats(integer) FROM public; + REVOKE EXECUTE ON FUNCTION pg_stat_reset_replication_slot(text) FROM public; REVOKE EXECUTE ON FUNCTION pg_stat_have_stats(text, oid, int8) FROM public; diff --git a/src/backend/utils/activity/Makefile b/src/backend/utils/activity/Makefile index b9fd66ea17..24b64a2742 100644 --- a/src/backend/utils/activity/Makefile +++ b/src/backend/utils/activity/Makefile @@ -20,6 +20,7 @@ OBJS = \ backend_status.o \ pgstat.o \ pgstat_archiver.o \ + pgstat_backend.o \ pgstat_bgwriter.o \ pgstat_checkpointer.o \ pgstat_database.o \ diff --git a/src/backend/utils/activity/backend_status.c b/src/backend/utils/activity/backend_status.c index 22c6dc378c..bf33e33a4e 100644 --- a/src/backend/utils/activity/backend_status.c +++ b/src/backend/utils/activity/backend_status.c @@ -426,6 +426,10 @@ pgstat_bestart(void) PGSTAT_END_WRITE_ACTIVITY(vbeentry); + /* Create the backend statistics entry */ + if (pgstat_tracks_backend_bktype(MyBackendType)) + pgstat_create_backend(MyProcNumber); + /* Update app name to current GUC setting */ if (application_name) pgstat_report_appname(application_name); diff --git a/src/backend/utils/activity/meson.build b/src/backend/utils/activity/meson.build index f73c22905c..380d3dd70c 100644 --- a/src/backend/utils/activity/meson.build +++ b/src/backend/utils/activity/meson.build @@ -5,6 +5,7 @@ backend_sources += files( 'backend_status.c', 'pgstat.c', 'pgstat_archiver.c', + 'pgstat_backend.c', 'pgstat_bgwriter.c', 'pgstat_checkpointer.c', 'pgstat_database.c', diff --git a/src/backend/utils/activity/pgstat.c b/src/backend/utils/activity/pgstat.c index b4e357c8a4..b72c779b2c 100644 --- a/src/backend/utils/activity/pgstat.c +++ b/src/backend/utils/activity/pgstat.c @@ -77,6 +77,7 @@ * * Each statistics kind is handled in a dedicated file: * - pgstat_archiver.c + * - pgstat_backend.c * - pgstat_bgwriter.c * - pgstat_checkpointer.c * - pgstat_database.c @@ -358,6 +359,22 @@ static const PgStat_KindInfo pgstat_kind_builtin_infos[PGSTAT_KIND_BUILTIN_SIZE] .reset_timestamp_cb = pgstat_subscription_reset_timestamp_cb, }, + [PGSTAT_KIND_BACKEND] = { + .name = "backend", + + .fixed_amount = false, + .write_to_file = false, + + .accessed_across_databases = true, + + .shared_size = sizeof(PgStatShared_Backend), + .shared_data_off = offsetof(PgStatShared_Backend, stats), + .shared_data_len = sizeof(((PgStatShared_Backend *) 0)->stats), + .pending_size = sizeof(PgStat_BackendPendingIO), + + .flush_pending_cb = pgstat_backend_flush_cb, + .reset_timestamp_cb = pgstat_backend_reset_timestamp_cb, + }, /* stats for fixed-numbered (mostly 1) objects */ @@ -602,6 +619,10 @@ pgstat_shutdown_hook(int code, Datum arg) Assert(dlist_is_empty(&pgStatPending)); dlist_init(&pgStatPending); + /* drop the backend stats entry */ + if (!pgstat_drop_entry(PGSTAT_KIND_BACKEND, InvalidOid, MyProcNumber)) + pgstat_request_entry_refs_gc(); + pgstat_detach_shmem(); #ifdef USE_ASSERT_CHECKING diff --git a/src/backend/utils/activity/pgstat_backend.c b/src/backend/utils/activity/pgstat_backend.c new file mode 100644 index 0000000000..6b2c9baa8c --- /dev/null +++ b/src/backend/utils/activity/pgstat_backend.c @@ -0,0 +1,188 @@ +/* ------------------------------------------------------------------------- + * + * pgstat_backend.c + * Implementation of backend statistics. + * + * This file contains the implementation of backend statistics. It is kept + * separate from pgstat.c to enforce the line between the statistics access / + * storage implementation and the details about individual types of + * statistics. + * + * This statistics kind uses a proc number as object ID for the hash table + * of pgstats. Entries are created each time a process is spawned, and are + * dropped when the process exits. These are not written to the pgstats file + * on disk. + * + * Copyright (c) 2001-2024, PostgreSQL Global Development Group + * + * IDENTIFICATION + * src/backend/utils/activity/pgstat_backend.c + * ------------------------------------------------------------------------- + */ + +#include "postgres.h" + +#include "utils/pgstat_internal.h" + +/* + * Returns statistics of a backend by proc number. + */ +PgStat_Backend * +pgstat_fetch_stat_backend(ProcNumber procNumber) +{ + PgStat_Backend *backend_entry; + + backend_entry = (PgStat_Backend *) pgstat_fetch_entry(PGSTAT_KIND_BACKEND, + InvalidOid, procNumber); + + return backend_entry; +} + +/* + * Flush out locally pending backend statistics + * + * If no stats have been recorded, this function returns false. + */ +bool +pgstat_backend_flush_cb(PgStat_EntryRef *entry_ref, bool nowait) +{ + PgStatShared_Backend *shbackendioent; + PgStat_BackendPendingIO *pendingent; + PgStat_BktypeIO *bktype_shstats; + + if (!pgstat_lock_entry(entry_ref, nowait)) + return false; + + shbackendioent = (PgStatShared_Backend *) entry_ref->shared_stats; + bktype_shstats = &shbackendioent->stats.stats; + pendingent = (PgStat_BackendPendingIO *) entry_ref->pending; + + for (int io_object = 0; io_object < IOOBJECT_NUM_TYPES; io_object++) + { + for (int io_context = 0; io_context < IOCONTEXT_NUM_TYPES; io_context++) + { + for (int io_op = 0; io_op < IOOP_NUM_TYPES; io_op++) + { + instr_time time; + + bktype_shstats->counts[io_object][io_context][io_op] += + pendingent->counts[io_object][io_context][io_op]; + + time = pendingent->pending_times[io_object][io_context][io_op]; + + bktype_shstats->times[io_object][io_context][io_op] += + INSTR_TIME_GET_MICROSEC(time); + } + } + } + + pgstat_unlock_entry(entry_ref); + + return true; +} + +/* + * Simpler wrapper of pgstat_backend_flush_cb() + */ +void +pgstat_flush_backend(bool nowait) +{ + PgStat_EntryRef *entry_ref; + + if (!pgstat_tracks_backend_bktype(MyBackendType)) + return; + + entry_ref = pgstat_get_entry_ref(PGSTAT_KIND_BACKEND, InvalidOid, + MyProcNumber, false, NULL); + (void) pgstat_backend_flush_cb(entry_ref, nowait); +} + +/* + * Create backend statistics entry for proc number. + */ +void +pgstat_create_backend(ProcNumber procnum) +{ + PgStat_EntryRef *entry_ref; + PgStatShared_Backend *shstatent; + + entry_ref = pgstat_prep_pending_entry(PGSTAT_KIND_BACKEND, InvalidOid, + procnum, NULL); + + shstatent = (PgStatShared_Backend *) entry_ref->shared_stats; + + /* + * NB: need to accept that there might be stats from an older backend, + * e.g. if we previously used this proc number. + */ + memset(&shstatent->stats, 0, sizeof(shstatent->stats)); +} + +/* + * Find or create a local PgStat_BackendPendingIO entry for proc number. + */ +PgStat_BackendPendingIO * +pgstat_prep_backend_pending(ProcNumber procnum) +{ + PgStat_EntryRef *entry_ref; + + entry_ref = pgstat_prep_pending_entry(PGSTAT_KIND_BACKEND, InvalidOid, + procnum, NULL); + + return entry_ref->pending; +} + +/* + * Backend statistics are not collected for all BackendTypes. + * + * The following BackendTypes do not participate in the backend stats + * subsystem: + * - The same and for the same reasons as in pgstat_tracks_io_bktype(). + * - B_BG_WRITER, B_CHECKPOINTER, B_STARTUP and B_AUTOVAC_LAUNCHER because their + * I/O stats are already visible in pg_stat_io and there is only one of those. + * + * Function returns true if BackendType participates in the backend stats + * subsystem and false if it does not. + * + * When adding a new BackendType, also consider adding relevant restrictions to + * pgstat_tracks_io_object() and pgstat_tracks_io_op(). + */ +bool +pgstat_tracks_backend_bktype(BackendType bktype) +{ + /* + * List every type so that new backend types trigger a warning about + * needing to adjust this switch. + */ + switch (bktype) + { + case B_INVALID: + case B_AUTOVAC_LAUNCHER: + case B_DEAD_END_BACKEND: + case B_ARCHIVER: + case B_LOGGER: + case B_WAL_RECEIVER: + case B_WAL_WRITER: + case B_WAL_SUMMARIZER: + case B_BG_WRITER: + case B_CHECKPOINTER: + case B_STARTUP: + return false; + + case B_AUTOVAC_WORKER: + case B_BACKEND: + case B_BG_WORKER: + case B_STANDALONE_BACKEND: + case B_SLOTSYNC_WORKER: + case B_WAL_SENDER: + return true; + } + + return false; +} + +void +pgstat_backend_reset_timestamp_cb(PgStatShared_Common *header, TimestampTz ts) +{ + ((PgStatShared_Backend *) header)->stats.stat_reset_timestamp = ts; +} diff --git a/src/backend/utils/activity/pgstat_io.c b/src/backend/utils/activity/pgstat_io.c index f9883af2b3..011a3326da 100644 --- a/src/backend/utils/activity/pgstat_io.c +++ b/src/backend/utils/activity/pgstat_io.c @@ -20,14 +20,6 @@ #include "storage/bufmgr.h" #include "utils/pgstat_internal.h" - -typedef struct PgStat_PendingIO -{ - PgStat_Counter counts[IOOBJECT_NUM_TYPES][IOCONTEXT_NUM_TYPES][IOOP_NUM_TYPES]; - instr_time pending_times[IOOBJECT_NUM_TYPES][IOCONTEXT_NUM_TYPES][IOOP_NUM_TYPES]; -} PgStat_PendingIO; - - static PgStat_PendingIO PendingIOStats; static bool have_iostats = false; @@ -87,6 +79,14 @@ pgstat_count_io_op_n(IOObject io_object, IOContext io_context, IOOp io_op, uint3 Assert((unsigned int) io_op < IOOP_NUM_TYPES); Assert(pgstat_tracks_io_op(MyBackendType, io_object, io_context, io_op)); + if (pgstat_tracks_backend_bktype(MyBackendType)) + { + PgStat_PendingIO *entry_ref; + + entry_ref = pgstat_prep_backend_pending(MyProcNumber); + entry_ref->counts[io_object][io_context][io_op] += cnt; + } + PendingIOStats.counts[io_object][io_context][io_op] += cnt; have_iostats = true; @@ -148,6 +148,15 @@ pgstat_count_io_op_time(IOObject io_object, IOContext io_context, IOOp io_op, INSTR_TIME_ADD(PendingIOStats.pending_times[io_object][io_context][io_op], io_time); + + if (pgstat_tracks_backend_bktype(MyBackendType)) + { + PgStat_PendingIO *entry_ref; + + entry_ref = pgstat_prep_backend_pending(MyProcNumber); + INSTR_TIME_ADD(entry_ref->pending_times[io_object][io_context][io_op], + io_time); + } } pgstat_count_io_op_n(io_object, io_context, io_op, cnt); diff --git a/src/backend/utils/activity/pgstat_relation.c b/src/backend/utils/activity/pgstat_relation.c index faba8b64d2..85e65557bb 100644 --- a/src/backend/utils/activity/pgstat_relation.c +++ b/src/backend/utils/activity/pgstat_relation.c @@ -264,6 +264,7 @@ pgstat_report_vacuum(Oid tableoid, bool shared, * VACUUM command has processed all tables and committed. */ pgstat_flush_io(false); + pgstat_flush_backend(false); } /* @@ -350,6 +351,7 @@ pgstat_report_analyze(Relation rel, /* see pgstat_report_vacuum() */ pgstat_flush_io(false); + pgstat_flush_backend(false); } /* diff --git a/src/backend/utils/adt/pgstatfuncs.c b/src/backend/utils/adt/pgstatfuncs.c index efd35f700a..8387e11a93 100644 --- a/src/backend/utils/adt/pgstatfuncs.c +++ b/src/backend/utils/adt/pgstatfuncs.c @@ -1274,8 +1274,9 @@ pg_stat_get_buf_alloc(PG_FUNCTION_ARGS) } /* -* When adding a new column to the pg_stat_io view, add a new enum value -* here above IO_NUM_COLUMNS. +* When adding a new column to the pg_stat_io view and the +* pg_stat_get_backend_io() function, add a new enum value here above +* IO_NUM_COLUMNS. */ typedef enum io_stat_col { @@ -1368,9 +1369,9 @@ pg_stat_us_to_ms(PgStat_Counter val_ms) /* * pg_stat_io_build_tuples * - * Helper routine for pg_stat_get_io() filling in a result tuplestore with one - * tuple for each object and each context supported by the caller, based on the - * contents of bktype_stats. + * Helper routine for pg_stat_get_io() and pg_stat_get_backend_io(), + * filling in a result tuplestore with one tuple for each object and each + * context supported by the caller, based on the contents of bktype_stats. */ static void pg_stat_io_build_tuples(ReturnSetInfo *rsinfo, @@ -1494,6 +1495,68 @@ pg_stat_get_io(PG_FUNCTION_ARGS) return (Datum) 0; } +/* + * Returns I/O statistics for a backend with given PID. + */ +Datum +pg_stat_get_backend_io(PG_FUNCTION_ARGS) +{ + ReturnSetInfo *rsinfo; + BackendType bktype; + int pid; + PGPROC *proc; + ProcNumber procNumber; + PgStat_Backend *backend_stats; + PgStat_BktypeIO *bktype_stats; + PgBackendStatus *beentry; + + InitMaterializedSRF(fcinfo, 0); + rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; + + pid = PG_GETARG_INT32(0); + proc = BackendPidGetProc(pid); + + /* + * This could be an auxiliary process but these do not report backend + * statistics due to pgstat_tracks_backend_bktype(), so there is no need + * for an extra call to AuxiliaryPidGetProc(). + */ + if (!proc) + return (Datum) 0; + + procNumber = GetNumberFromPGProc(proc); + + backend_stats = pgstat_fetch_stat_backend(procNumber); + + if (!backend_stats) + return (Datum) 0; + + beentry = pgstat_get_beentry_by_proc_number(procNumber); + bktype = beentry->st_backendType; + + /* if PID does not match, leave */ + if (beentry->st_procpid != pid) + return (Datum) 0; + + /* backend may be gone, so recheck in case */ + if (bktype == B_INVALID) + return (Datum) 0; + + bktype_stats = &backend_stats->stats; + + /* + * In Assert builds, we can afford an extra loop through all of the + * counters (in pg_stat_io_build_tuples()), checking that only expected + * stats are non-zero, since it keeps the non-Assert code cleaner. + */ + Assert(pgstat_bktype_io_stats_valid(bktype_stats, bktype)); + + /* save tuples with data from this PgStat_BktypeIO */ + pg_stat_io_build_tuples(rsinfo, bktype_stats, bktype, + backend_stats->stat_reset_timestamp); + return (Datum) 0; +} + /* * Returns statistics of WAL activity */ @@ -1799,6 +1862,30 @@ pg_stat_reset_single_function_counters(PG_FUNCTION_ARGS) PG_RETURN_VOID(); } +/* + * Reset statistics of backend with given PID. + */ +Datum +pg_stat_reset_backend_stats(PG_FUNCTION_ARGS) +{ + PGPROC *proc; + int backend_pid = PG_GETARG_INT32(0); + + proc = BackendPidGetProc(backend_pid); + + /* + * This could be an auxiliary process but these do not report backend + * statistics due to pgstat_tracks_backend_bktype(), so there is no need + * for an extra call to AuxiliaryPidGetProc(). + */ + if (!proc) + PG_RETURN_VOID(); + + pgstat_reset(PGSTAT_KIND_BACKEND, InvalidOid, GetNumberFromPGProc(proc)); + + PG_RETURN_VOID(); +} + /* Reset SLRU counters (a specific one or all of them). */ Datum pg_stat_reset_slru(PG_FUNCTION_ARGS) diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index 0f22c21723..2dcc2d42da 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -5913,6 +5913,15 @@ proargnames => '{backend_type,object,context,reads,read_time,writes,write_time,writebacks,writeback_time,extends,extend_time,op_bytes,hits,evictions,reuses,fsyncs,fsync_time,stats_reset}', prosrc => 'pg_stat_get_io' }, +{ oid => '8806', descr => 'statistics: backend IO statistics', + proname => 'pg_stat_get_backend_io', prorows => '5', proretset => 't', + provolatile => 'v', proparallel => 'r', prorettype => 'record', + proargtypes => 'int4', + proallargtypes => '{int4,text,text,text,int8,float8,int8,float8,int8,float8,int8,float8,int8,int8,int8,int8,int8,float8,timestamptz}', + proargmodes => '{i,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o}', + proargnames => '{backend_pid,backend_type,object,context,reads,read_time,writes,write_time,writebacks,writeback_time,extends,extend_time,op_bytes,hits,evictions,reuses,fsyncs,fsync_time,stats_reset}', + prosrc => 'pg_stat_get_backend_io' }, + { oid => '1136', descr => 'statistics: information about WAL activity', proname => 'pg_stat_get_wal', proisstrict => 'f', provolatile => 's', proparallel => 'r', prorettype => 'record', proargtypes => '', @@ -6052,6 +6061,10 @@ proname => 'pg_stat_reset_single_function_counters', provolatile => 'v', prorettype => 'void', proargtypes => 'oid', prosrc => 'pg_stat_reset_single_function_counters' }, +{ oid => '8807', descr => 'statistics: reset statistics for a single backend', + proname => 'pg_stat_reset_backend_stats', provolatile => 'v', + prorettype => 'void', proargtypes => 'int4', + prosrc => 'pg_stat_reset_backend_stats' }, { oid => '2307', descr => 'statistics: reset collected statistics for a single SLRU', proname => 'pg_stat_reset_slru', proisstrict => 'f', provolatile => 'v', diff --git a/src/include/pgstat.h b/src/include/pgstat.h index ebfeef2f46..56ed1b8bb3 100644 --- a/src/include/pgstat.h +++ b/src/include/pgstat.h @@ -49,14 +49,15 @@ #define PGSTAT_KIND_FUNCTION 3 /* per-function statistics */ #define PGSTAT_KIND_REPLSLOT 4 /* per-slot statistics */ #define PGSTAT_KIND_SUBSCRIPTION 5 /* per-subscription statistics */ +#define PGSTAT_KIND_BACKEND 6 /* per-backend statistics */ /* stats for fixed-numbered objects */ -#define PGSTAT_KIND_ARCHIVER 6 -#define PGSTAT_KIND_BGWRITER 7 -#define PGSTAT_KIND_CHECKPOINTER 8 -#define PGSTAT_KIND_IO 9 -#define PGSTAT_KIND_SLRU 10 -#define PGSTAT_KIND_WAL 11 +#define PGSTAT_KIND_ARCHIVER 7 +#define PGSTAT_KIND_BGWRITER 8 +#define PGSTAT_KIND_CHECKPOINTER 9 +#define PGSTAT_KIND_IO 10 +#define PGSTAT_KIND_SLRU 11 +#define PGSTAT_KIND_WAL 12 #define PGSTAT_KIND_BUILTIN_MIN PGSTAT_KIND_DATABASE #define PGSTAT_KIND_BUILTIN_MAX PGSTAT_KIND_WAL @@ -362,12 +363,26 @@ typedef struct PgStat_BktypeIO PgStat_Counter times[IOOBJECT_NUM_TYPES][IOCONTEXT_NUM_TYPES][IOOP_NUM_TYPES]; } PgStat_BktypeIO; +typedef struct PgStat_PendingIO +{ + PgStat_Counter counts[IOOBJECT_NUM_TYPES][IOCONTEXT_NUM_TYPES][IOOP_NUM_TYPES]; + instr_time pending_times[IOOBJECT_NUM_TYPES][IOCONTEXT_NUM_TYPES][IOOP_NUM_TYPES]; +} PgStat_PendingIO; + typedef struct PgStat_IO { TimestampTz stat_reset_timestamp; PgStat_BktypeIO stats[BACKEND_NUM_TYPES]; } PgStat_IO; +/* Backend statistics store the same amount of IO data as PGSTAT_KIND_IO */ +typedef PgStat_PendingIO PgStat_BackendPendingIO; + +typedef struct PgStat_Backend +{ + TimestampTz stat_reset_timestamp; + PgStat_BktypeIO stats; +} PgStat_Backend; typedef struct PgStat_StatDBEntry { @@ -549,6 +564,13 @@ extern bool pgstat_have_entry(PgStat_Kind kind, Oid dboid, uint64 objid); extern void pgstat_report_archiver(const char *xlog, bool failed); extern PgStat_ArchiverStats *pgstat_fetch_stat_archiver(void); +/* + * Functions in pgstat_backend.c + */ + +extern PgStat_Backend *pgstat_fetch_stat_backend(ProcNumber procNumber); +extern bool pgstat_tracks_backend_bktype(BackendType bktype); +extern void pgstat_create_backend(ProcNumber procnum); /* * Functions in pgstat_bgwriter.c diff --git a/src/include/utils/pgstat_internal.h b/src/include/utils/pgstat_internal.h index 7338bc1e28..811ed9b005 100644 --- a/src/include/utils/pgstat_internal.h +++ b/src/include/utils/pgstat_internal.h @@ -450,6 +450,11 @@ typedef struct PgStatShared_ReplSlot PgStat_StatReplSlotEntry stats; } PgStatShared_ReplSlot; +typedef struct PgStatShared_Backend +{ + PgStatShared_Common header; + PgStat_Backend stats; +} PgStatShared_Backend; /* * Central shared memory entry for the cumulative stats system. @@ -604,6 +609,15 @@ extern void pgstat_archiver_init_shmem_cb(void *stats); extern void pgstat_archiver_reset_all_cb(TimestampTz ts); extern void pgstat_archiver_snapshot_cb(void); +/* + * Functions in pgstat_backend.c + */ + +extern void pgstat_flush_backend(bool nowait); + +extern PgStat_BackendPendingIO *pgstat_prep_backend_pending(ProcNumber procnum); +extern bool pgstat_backend_flush_cb(PgStat_EntryRef *entry_ref, bool nowait); +extern void pgstat_backend_reset_timestamp_cb(PgStatShared_Common *header, TimestampTz ts); /* * Functions in pgstat_bgwriter.c diff --git a/src/test/regress/expected/stats.out b/src/test/regress/expected/stats.out index 56771f83ed..150b6dcf74 100644 --- a/src/test/regress/expected/stats.out +++ b/src/test/regress/expected/stats.out @@ -1249,7 +1249,8 @@ SELECT pg_stat_get_subscription_stats(NULL); (1 row) --- Test that the following operations are tracked in pg_stat_io: +-- Test that the following operations are tracked in pg_stat_io and in +-- backend stats: -- - reads of target blocks into shared buffers -- - writes of shared buffers to permanent storage -- - extends of relations using shared buffers @@ -1259,11 +1260,19 @@ SELECT pg_stat_get_subscription_stats(NULL); -- be sure of the state of shared buffers at the point the test is run. -- Create a regular table and insert some data to generate IOCONTEXT_NORMAL -- extends. +SELECT pid AS checkpointer_pid FROM pg_stat_activity + WHERE backend_type = 'checkpointer' \gset SELECT sum(extends) AS io_sum_shared_before_extends FROM pg_stat_io WHERE context = 'normal' AND object = 'relation' \gset +SELECT sum(extends) AS my_io_sum_shared_before_extends + FROM pg_stat_get_backend_io(pg_backend_pid()) + WHERE context = 'normal' AND object = 'relation' \gset SELECT sum(writes) AS writes, sum(fsyncs) AS fsyncs FROM pg_stat_io WHERE object = 'relation' \gset io_sum_shared_before_ +SELECT sum(writes) AS writes, sum(fsyncs) AS fsyncs + FROM pg_stat_get_backend_io(pg_backend_pid()) + WHERE object = 'relation' \gset my_io_sum_shared_before_ CREATE TABLE test_io_shared(a int); INSERT INTO test_io_shared SELECT i FROM generate_series(1,100)i; SELECT pg_stat_force_next_flush(); @@ -1280,8 +1289,17 @@ SELECT :io_sum_shared_after_extends > :io_sum_shared_before_extends; t (1 row) +SELECT sum(extends) AS my_io_sum_shared_after_extends + FROM pg_stat_get_backend_io(pg_backend_pid()) + WHERE context = 'normal' AND object = 'relation' \gset +SELECT :my_io_sum_shared_after_extends > :my_io_sum_shared_before_extends; + ?column? +---------- + t +(1 row) + -- After a checkpoint, there should be some additional IOCONTEXT_NORMAL writes --- and fsyncs. +-- and fsyncs in the global stats (not for the backend). -- See comment above for rationale for two explicit CHECKPOINTs. CHECKPOINT; CHECKPOINT; @@ -1301,6 +1319,23 @@ SELECT current_setting('fsync') = 'off' t (1 row) +SELECT sum(writes) AS writes, sum(fsyncs) AS fsyncs + FROM pg_stat_get_backend_io(pg_backend_pid()) + WHERE object = 'relation' \gset my_io_sum_shared_after_ +SELECT :my_io_sum_shared_after_writes >= :my_io_sum_shared_before_writes; + ?column? +---------- + t +(1 row) + +SELECT current_setting('fsync') = 'off' + OR (:my_io_sum_shared_after_fsyncs = :my_io_sum_shared_before_fsyncs + AND :my_io_sum_shared_after_fsyncs= 0); + ?column? +---------- + t +(1 row) + -- Change the tablespace so that the table is rewritten directly, then SELECT -- from it to cause it to be read back into shared buffers. SELECT sum(reads) AS io_sum_shared_before_reads @@ -1521,6 +1556,8 @@ SELECT pg_stat_have_stats('io', 0, 0); SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS io_stats_pre_reset FROM pg_stat_io \gset +SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS my_io_stats_pre_reset + FROM pg_stat_get_backend_io(pg_backend_pid()) \gset SELECT pg_stat_reset_shared('io'); pg_stat_reset_shared ---------------------- @@ -1535,6 +1572,47 @@ SELECT :io_stats_post_reset < :io_stats_pre_reset; t (1 row) +SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS my_io_stats_post_reset + FROM pg_stat_get_backend_io(pg_backend_pid()) \gset +-- pg_stat_reset_shared() did not reset backend IO stats +SELECT :my_io_stats_pre_reset <= :my_io_stats_post_reset; + ?column? +---------- + t +(1 row) + +-- but pg_stat_reset_backend_stats() does +SELECT pg_stat_reset_backend_stats(pg_backend_pid()); + pg_stat_reset_backend_stats +----------------------------- + +(1 row) + +SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS my_io_stats_post_backend_reset + FROM pg_stat_get_backend_io(pg_backend_pid()) \gset +SELECT :my_io_stats_pre_reset > :my_io_stats_post_backend_reset; + ?column? +---------- + t +(1 row) + +-- Check invalid input for pg_stat_get_backend_io() +SELECT pg_stat_get_backend_io(NULL); + pg_stat_get_backend_io +------------------------ +(0 rows) + +SELECT pg_stat_get_backend_io(0); + pg_stat_get_backend_io +------------------------ +(0 rows) + +-- Auxiliary processes return no data. +SELECT pg_stat_get_backend_io(:checkpointer_pid); + pg_stat_get_backend_io +------------------------ +(0 rows) + -- test BRIN index doesn't block HOT update CREATE TABLE brin_hot ( id integer PRIMARY KEY, diff --git a/src/test/regress/sql/stats.sql b/src/test/regress/sql/stats.sql index 7147cc2f89..1e7d0ff665 100644 --- a/src/test/regress/sql/stats.sql +++ b/src/test/regress/sql/stats.sql @@ -595,7 +595,8 @@ SELECT pg_stat_get_replication_slot(NULL); SELECT pg_stat_get_subscription_stats(NULL); --- Test that the following operations are tracked in pg_stat_io: +-- Test that the following operations are tracked in pg_stat_io and in +-- backend stats: -- - reads of target blocks into shared buffers -- - writes of shared buffers to permanent storage -- - extends of relations using shared buffers @@ -607,20 +608,32 @@ SELECT pg_stat_get_subscription_stats(NULL); -- Create a regular table and insert some data to generate IOCONTEXT_NORMAL -- extends. +SELECT pid AS checkpointer_pid FROM pg_stat_activity + WHERE backend_type = 'checkpointer' \gset SELECT sum(extends) AS io_sum_shared_before_extends FROM pg_stat_io WHERE context = 'normal' AND object = 'relation' \gset +SELECT sum(extends) AS my_io_sum_shared_before_extends + FROM pg_stat_get_backend_io(pg_backend_pid()) + WHERE context = 'normal' AND object = 'relation' \gset SELECT sum(writes) AS writes, sum(fsyncs) AS fsyncs FROM pg_stat_io WHERE object = 'relation' \gset io_sum_shared_before_ +SELECT sum(writes) AS writes, sum(fsyncs) AS fsyncs + FROM pg_stat_get_backend_io(pg_backend_pid()) + WHERE object = 'relation' \gset my_io_sum_shared_before_ CREATE TABLE test_io_shared(a int); INSERT INTO test_io_shared SELECT i FROM generate_series(1,100)i; SELECT pg_stat_force_next_flush(); SELECT sum(extends) AS io_sum_shared_after_extends FROM pg_stat_io WHERE context = 'normal' AND object = 'relation' \gset SELECT :io_sum_shared_after_extends > :io_sum_shared_before_extends; +SELECT sum(extends) AS my_io_sum_shared_after_extends + FROM pg_stat_get_backend_io(pg_backend_pid()) + WHERE context = 'normal' AND object = 'relation' \gset +SELECT :my_io_sum_shared_after_extends > :my_io_sum_shared_before_extends; -- After a checkpoint, there should be some additional IOCONTEXT_NORMAL writes --- and fsyncs. +-- and fsyncs in the global stats (not for the backend). -- See comment above for rationale for two explicit CHECKPOINTs. CHECKPOINT; CHECKPOINT; @@ -630,6 +643,13 @@ SELECT sum(writes) AS writes, sum(fsyncs) AS fsyncs SELECT :io_sum_shared_after_writes > :io_sum_shared_before_writes; SELECT current_setting('fsync') = 'off' OR :io_sum_shared_after_fsyncs > :io_sum_shared_before_fsyncs; +SELECT sum(writes) AS writes, sum(fsyncs) AS fsyncs + FROM pg_stat_get_backend_io(pg_backend_pid()) + WHERE object = 'relation' \gset my_io_sum_shared_after_ +SELECT :my_io_sum_shared_after_writes >= :my_io_sum_shared_before_writes; +SELECT current_setting('fsync') = 'off' + OR (:my_io_sum_shared_after_fsyncs = :my_io_sum_shared_before_fsyncs + AND :my_io_sum_shared_after_fsyncs= 0); -- Change the tablespace so that the table is rewritten directly, then SELECT -- from it to cause it to be read back into shared buffers. @@ -762,11 +782,27 @@ SELECT :io_sum_bulkwrite_strategy_extends_after > :io_sum_bulkwrite_strategy_ext SELECT pg_stat_have_stats('io', 0, 0); SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS io_stats_pre_reset FROM pg_stat_io \gset +SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS my_io_stats_pre_reset + FROM pg_stat_get_backend_io(pg_backend_pid()) \gset SELECT pg_stat_reset_shared('io'); SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS io_stats_post_reset FROM pg_stat_io \gset SELECT :io_stats_post_reset < :io_stats_pre_reset; - +SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS my_io_stats_post_reset + FROM pg_stat_get_backend_io(pg_backend_pid()) \gset +-- pg_stat_reset_shared() did not reset backend IO stats +SELECT :my_io_stats_pre_reset <= :my_io_stats_post_reset; +-- but pg_stat_reset_backend_stats() does +SELECT pg_stat_reset_backend_stats(pg_backend_pid()); +SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS my_io_stats_post_backend_reset + FROM pg_stat_get_backend_io(pg_backend_pid()) \gset +SELECT :my_io_stats_pre_reset > :my_io_stats_post_backend_reset; + +-- Check invalid input for pg_stat_get_backend_io() +SELECT pg_stat_get_backend_io(NULL); +SELECT pg_stat_get_backend_io(0); +-- Auxiliary processes return no data. +SELECT pg_stat_get_backend_io(:checkpointer_pid); -- test BRIN index doesn't block HOT update CREATE TABLE brin_hot ( diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index ce33e55bf1..398dd92527 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2121,6 +2121,7 @@ PgFdwSamplingMethod PgFdwScanState PgIfAddrCallback PgStatShared_Archiver +PgStatShared_Backend PgStatShared_BgWriter PgStatShared_Checkpointer PgStatShared_Common @@ -2136,6 +2137,8 @@ PgStatShared_SLRU PgStatShared_Subscription PgStatShared_Wal PgStat_ArchiverStats +PgStat_Backend +PgStat_BackendPendingIO PgStat_BackendSubEntry PgStat_BgWriterStats PgStat_BktypeIO -- 2.34.1 --YrW53rPX+W0tiwfU-- ^ permalink raw reply [nested|flat] 25+ messages in thread
* [PATCH v3 3/3] Better express platform requirements in s_lock.h. @ 2026-05-07 20:32 Nathan Bossart <[email protected]> 0 siblings, 0 replies; 25+ messages in thread From: Nathan Bossart @ 2026-05-07 20:32 UTC (permalink / raw) --- src/include/storage/s_lock.h | 56 ++++++++++++++++++++---------------- 1 file changed, 32 insertions(+), 24 deletions(-) diff --git a/src/include/storage/s_lock.h b/src/include/storage/s_lock.h index fb872edd2f0..c4369ee4ff6 100644 --- a/src/include/storage/s_lock.h +++ b/src/include/storage/s_lock.h @@ -44,23 +44,16 @@ * atomic test-and-set only when it appears free. * * TAS() and TAS_SPIN() are NOT part of the API, and should never be called - * directly. - * - * CAUTION: on some platforms TAS() and/or TAS_SPIN() may sometimes report - * failure to acquire a lock even when the lock is not locked. For example, - * on Alpha TAS() will "fail" if interrupted. Therefore a retry loop must - * always be used, even if you are certain the lock is free. + * directly. If a platform-specific TAS() is defined, the platform must + * _not_ define its own S_LOCK(). Conversely, if a platform-specific + * S_LOCK() is defined, the platform must _not_ define its own TAS(), but + * it does need to define its own TAS_SPIN(). Currently, all supported + * platforms define TAS() and use the default S_LOCK() implementation, so + * that is probably a good place to start if adding a new one. * * It is the responsibility of these macros to make sure that the compiler * does not re-order accesses to shared memory to precede the actual lock - * acquisition, or follow the lock release. Prior to PostgreSQL 9.5, this - * was the caller's responsibility, which meant that callers had to use - * volatile-qualified pointers to refer to both the spinlock itself and the - * shared data being accessed within the spinlocked critical section. This - * was notationally awkward, easy to forget (and thus error-prone), and - * prevented some useful compiler optimizations. For these reasons, we - * now require that the macros themselves prevent compiler re-ordering, - * so that the caller doesn't need to take special precautions. + * acquisition, or follow the lock release. * * On platforms with weak memory ordering, the TAS(), TAS_SPIN(), and * S_UNLOCK() macros must further include hardware-level memory fence @@ -72,7 +65,7 @@ * * On most supported platforms, TAS() uses a tas() function written * in assembly language to execute a hardware atomic-test-and-set - * instruction. Equivalent OS-supplied mutex routines could be used too. + * instruction. Equivalent compiler intrinsics are another popular option. * * * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group @@ -642,20 +635,25 @@ spin_delay(void) #endif /* !defined(TAS) */ -/* Blow up if we didn't have any way to do spinlocks */ -#ifndef TAS -#error PostgreSQL does not have spinlock support on this platform. Please report this to [email protected]. -#endif - - /* * Default Definitions - override these above as needed. */ -#if !defined(S_LOCK) +/* + * Make sure S_LOCK is defined, either explicitly for the platform or via a TAS + * macro for the platform. Exactly one of either S_LOCK or TAS should be + * defined for a supported platform at this point in the file. + */ +#if defined(S_LOCK) +#if defined(TAS) +#error Both TAS and S_LOCK defined on this platform. Please report this to [email protected]. +#endif +#elif defined(TAS) #define S_LOCK(lock) \ (TAS(lock) ? s_lock((lock), __FILE__, __LINE__, __func__) : 0) -#endif /* S_LOCK */ +#else +#error Neither TAS nor S_LOCK defined on this platform. Please report this to [email protected]. +#endif #if !defined(S_UNLOCK) /* @@ -687,9 +685,19 @@ extern void s_unlock(volatile slock_t *lock); #define SPIN_DELAY() ((void) 0) #endif /* SPIN_DELAY */ +/* + * We can only define TAS_SPIN if TAS was defined. Otherwise, the platform + * defined its own S_LOCK without TAS, and therefore is responsible for + * defining its own TAS_SPIN as well. (Note that we currently do not have any + * platforms that don't define TAS.) + */ #if !defined(TAS_SPIN) +#if defined(TAS) #define TAS_SPIN(lock) TAS(lock) -#endif /* TAS_SPIN */ +#else +#error Neither TAS nor TAS_SPIN defined on this platform. Please report this to [email protected]. +#endif /* TAS */ +#endif /* ! TAS_SPIN */ /* -- 2.50.1 (Apple Git-155) --IHFkEDr8CkxIje3R-- ^ permalink raw reply [nested|flat] 25+ messages in thread
* [PATCH v4 3/3] Better express platform requirements in s_lock.h. @ 2026-05-07 20:32 Nathan Bossart <[email protected]> 0 siblings, 0 replies; 25+ messages in thread From: Nathan Bossart @ 2026-05-07 20:32 UTC (permalink / raw) --- src/include/storage/s_lock.h | 56 ++++++++++++++++++++---------------- 1 file changed, 32 insertions(+), 24 deletions(-) diff --git a/src/include/storage/s_lock.h b/src/include/storage/s_lock.h index fb872edd2f0..c4369ee4ff6 100644 --- a/src/include/storage/s_lock.h +++ b/src/include/storage/s_lock.h @@ -44,23 +44,16 @@ * atomic test-and-set only when it appears free. * * TAS() and TAS_SPIN() are NOT part of the API, and should never be called - * directly. - * - * CAUTION: on some platforms TAS() and/or TAS_SPIN() may sometimes report - * failure to acquire a lock even when the lock is not locked. For example, - * on Alpha TAS() will "fail" if interrupted. Therefore a retry loop must - * always be used, even if you are certain the lock is free. + * directly. If a platform-specific TAS() is defined, the platform must + * _not_ define its own S_LOCK(). Conversely, if a platform-specific + * S_LOCK() is defined, the platform must _not_ define its own TAS(), but + * it does need to define its own TAS_SPIN(). Currently, all supported + * platforms define TAS() and use the default S_LOCK() implementation, so + * that is probably a good place to start if adding a new one. * * It is the responsibility of these macros to make sure that the compiler * does not re-order accesses to shared memory to precede the actual lock - * acquisition, or follow the lock release. Prior to PostgreSQL 9.5, this - * was the caller's responsibility, which meant that callers had to use - * volatile-qualified pointers to refer to both the spinlock itself and the - * shared data being accessed within the spinlocked critical section. This - * was notationally awkward, easy to forget (and thus error-prone), and - * prevented some useful compiler optimizations. For these reasons, we - * now require that the macros themselves prevent compiler re-ordering, - * so that the caller doesn't need to take special precautions. + * acquisition, or follow the lock release. * * On platforms with weak memory ordering, the TAS(), TAS_SPIN(), and * S_UNLOCK() macros must further include hardware-level memory fence @@ -72,7 +65,7 @@ * * On most supported platforms, TAS() uses a tas() function written * in assembly language to execute a hardware atomic-test-and-set - * instruction. Equivalent OS-supplied mutex routines could be used too. + * instruction. Equivalent compiler intrinsics are another popular option. * * * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group @@ -642,20 +635,25 @@ spin_delay(void) #endif /* !defined(TAS) */ -/* Blow up if we didn't have any way to do spinlocks */ -#ifndef TAS -#error PostgreSQL does not have spinlock support on this platform. Please report this to [email protected]. -#endif - - /* * Default Definitions - override these above as needed. */ -#if !defined(S_LOCK) +/* + * Make sure S_LOCK is defined, either explicitly for the platform or via a TAS + * macro for the platform. Exactly one of either S_LOCK or TAS should be + * defined for a supported platform at this point in the file. + */ +#if defined(S_LOCK) +#if defined(TAS) +#error Both TAS and S_LOCK defined on this platform. Please report this to [email protected]. +#endif +#elif defined(TAS) #define S_LOCK(lock) \ (TAS(lock) ? s_lock((lock), __FILE__, __LINE__, __func__) : 0) -#endif /* S_LOCK */ +#else +#error Neither TAS nor S_LOCK defined on this platform. Please report this to [email protected]. +#endif #if !defined(S_UNLOCK) /* @@ -687,9 +685,19 @@ extern void s_unlock(volatile slock_t *lock); #define SPIN_DELAY() ((void) 0) #endif /* SPIN_DELAY */ +/* + * We can only define TAS_SPIN if TAS was defined. Otherwise, the platform + * defined its own S_LOCK without TAS, and therefore is responsible for + * defining its own TAS_SPIN as well. (Note that we currently do not have any + * platforms that don't define TAS.) + */ #if !defined(TAS_SPIN) +#if defined(TAS) #define TAS_SPIN(lock) TAS(lock) -#endif /* TAS_SPIN */ +#else +#error Neither TAS nor TAS_SPIN defined on this platform. Please report this to [email protected]. +#endif /* TAS */ +#endif /* ! TAS_SPIN */ /* -- 2.50.1 (Apple Git-155) --JmXY/Yn6swRVDyih-- ^ permalink raw reply [nested|flat] 25+ messages in thread
* [PATCH v5 3/3] Better express platform requirements in s_lock.h. @ 2026-05-07 20:32 Nathan Bossart <[email protected]> 0 siblings, 0 replies; 25+ messages in thread From: Nathan Bossart @ 2026-05-07 20:32 UTC (permalink / raw) --- src/backend/storage/lmgr/s_lock.c | 2 ++ src/include/storage/s_lock.h | 59 +++++++++++++++++-------------- 2 files changed, 35 insertions(+), 26 deletions(-) diff --git a/src/backend/storage/lmgr/s_lock.c b/src/backend/storage/lmgr/s_lock.c index 6df568eccb3..34c6de66773 100644 --- a/src/backend/storage/lmgr/s_lock.c +++ b/src/backend/storage/lmgr/s_lock.c @@ -91,6 +91,7 @@ s_lock_stuck(const char *file, int line, const char *func) #endif } +#ifdef USE_DEFAULT_S_LOCK /* * s_lock(lock) - platform-independent portion of waiting for a spinlock. */ @@ -110,6 +111,7 @@ s_lock(volatile slock_t *lock, const char *file, int line, const char *func) return delayStatus.delays; } +#endif #ifdef USE_DEFAULT_S_UNLOCK void diff --git a/src/include/storage/s_lock.h b/src/include/storage/s_lock.h index fb872edd2f0..65f00c06518 100644 --- a/src/include/storage/s_lock.h +++ b/src/include/storage/s_lock.h @@ -44,23 +44,16 @@ * atomic test-and-set only when it appears free. * * TAS() and TAS_SPIN() are NOT part of the API, and should never be called - * directly. - * - * CAUTION: on some platforms TAS() and/or TAS_SPIN() may sometimes report - * failure to acquire a lock even when the lock is not locked. For example, - * on Alpha TAS() will "fail" if interrupted. Therefore a retry loop must - * always be used, even if you are certain the lock is free. + * directly. If a platform-specific TAS() is defined, the platform must + * _not_ define its own S_LOCK(). Conversely, if a platform-specific + * S_LOCK() is defined, the platform must _not_ define its own TAS(). + * Currently, all supported platforms define TAS() and use the default + * S_LOCK() implementation, so that is probably a good place to start if + * adding a new one. * * It is the responsibility of these macros to make sure that the compiler * does not re-order accesses to shared memory to precede the actual lock - * acquisition, or follow the lock release. Prior to PostgreSQL 9.5, this - * was the caller's responsibility, which meant that callers had to use - * volatile-qualified pointers to refer to both the spinlock itself and the - * shared data being accessed within the spinlocked critical section. This - * was notationally awkward, easy to forget (and thus error-prone), and - * prevented some useful compiler optimizations. For these reasons, we - * now require that the macros themselves prevent compiler re-ordering, - * so that the caller doesn't need to take special precautions. + * acquisition, or follow the lock release. * * On platforms with weak memory ordering, the TAS(), TAS_SPIN(), and * S_UNLOCK() macros must further include hardware-level memory fence @@ -72,7 +65,7 @@ * * On most supported platforms, TAS() uses a tas() function written * in assembly language to execute a hardware atomic-test-and-set - * instruction. Equivalent OS-supplied mutex routines could be used too. + * instruction. Equivalent compiler intrinsics are another popular option. * * * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group @@ -642,20 +635,27 @@ spin_delay(void) #endif /* !defined(TAS) */ -/* Blow up if we didn't have any way to do spinlocks */ -#ifndef TAS -#error PostgreSQL does not have spinlock support on this platform. Please report this to [email protected]. -#endif - - /* * Default Definitions - override these above as needed. */ -#if !defined(S_LOCK) +/* + * Make sure S_LOCK is defined, either explicitly for the platform or via a TAS + * macro for the platform. Exactly one of either S_LOCK or TAS should be + * defined for a supported platform at this point in the file. + */ +#if defined(S_LOCK) +#if defined(TAS) +#error Both TAS and S_LOCK defined on this platform. Please report this to [email protected]. +#endif +#elif defined(TAS) +#define USE_DEFAULT_S_LOCK +extern int s_lock(volatile slock_t *lock, const char *file, int line, const char *func); #define S_LOCK(lock) \ (TAS(lock) ? s_lock((lock), __FILE__, __LINE__, __func__) : 0) -#endif /* S_LOCK */ +#else +#error Neither TAS nor S_LOCK defined on this platform. Please report this to [email protected]. +#endif #if !defined(S_UNLOCK) /* @@ -687,15 +687,22 @@ extern void s_unlock(volatile slock_t *lock); #define SPIN_DELAY() ((void) 0) #endif /* SPIN_DELAY */ -#if !defined(TAS_SPIN) +/* + * We can only define TAS_SPIN if TAS was defined. Otherwise, the platform + * defined its own S_LOCK without TAS. Since TAS_SPIN is only used by the + * default S_LOCK's helper function, there's no need to define TAS_SPIN at all + * in that case, unless you plan to use it in a platform-specific S_LOCK helper + * function. (Note that we currently do not have any platforms that don't + * define TAS.) + */ +#if !defined(TAS_SPIN) && defined(TAS) #define TAS_SPIN(lock) TAS(lock) -#endif /* TAS_SPIN */ +#endif /* ! TAS_SPIN && TAS */ /* * Platform-independent out-of-line support routines */ -extern int s_lock(volatile slock_t *lock, const char *file, int line, const char *func); /* Support for dynamic adjustment of spins_per_delay */ #define DEFAULT_SPINS_PER_DELAY 100 -- 2.50.1 (Apple Git-155) --EzTP03ol5OjsRjZr-- ^ permalink raw reply [nested|flat] 25+ messages in thread
end of thread, other threads:[~2026-05-07 20:32 UTC | newest] Thread overview: 25+ messages (download: mbox mbox.gz follow: Atom feed) -- links below jump to the message on this page -- 2019-02-14 13:37 [PATCH] Add a function to reset the cumulative statistics Pierre <[email protected]> 2020-02-23 03:17 [PATCH v7 9/9] Add explain(REGRESS)... Justin Pryzby <[email protected]> 2024-08-22 15:16 [PATCH v2 1/2] per backend I/O statistics Bertrand Drouvot <[email protected]> 2024-08-22 15:16 [PATCH v3 1/2] per backend I/O statistics Bertrand Drouvot <[email protected]> 2024-08-22 15:16 [PATCH v4 1/2] per backend I/O statistics Bertrand Drouvot <[email protected]> 2024-08-22 15:16 [PATCH v1 1/2] per backend I/O statistics Bertrand Drouvot <[email protected]> 2024-08-22 15:16 [PATCH v4 1/2] per backend I/O statistics Bertrand Drouvot <[email protected]> 2024-08-22 15:16 [PATCH v2 1/2] per backend I/O statistics Bertrand Drouvot <[email protected]> 2024-08-22 15:16 [PATCH v3 1/2] per backend I/O statistics Bertrand Drouvot <[email protected]> 2024-08-22 15:16 [PATCH v1 1/2] per backend I/O statistics Bertrand Drouvot <[email protected]> 2024-10-28 12:50 [PATCH v9] per backend I/O statistics Bertrand Drouvot <[email protected]> 2024-10-28 12:50 [PATCH v8] per backend I/O statistics Bertrand Drouvot <[email protected]> 2024-10-28 12:50 [PATCH v5 1/4] per backend I/O statistics Bertrand Drouvot <[email protected]> 2024-10-28 12:50 [PATCH v6] per backend I/O statistics Bertrand Drouvot <[email protected]> 2024-10-28 12:50 [PATCH v7] per backend I/O statistics Bertrand Drouvot <[email protected]> 2024-10-28 12:50 [PATCH v9] per backend I/O statistics Bertrand Drouvot <[email protected]> 2024-10-28 12:50 [PATCH v8] per backend I/O statistics Bertrand Drouvot <[email protected]> 2024-10-28 12:50 [PATCH v6] per backend I/O statistics Bertrand Drouvot <[email protected]> 2024-10-28 12:50 [PATCH v5 1/4] per backend I/O statistics Bertrand Drouvot <[email protected]> 2024-10-28 12:50 [PATCH v7] per backend I/O statistics Bertrand Drouvot <[email protected]> 2024-12-18 06:39 [PATCH v11 2/2] per backend I/O statistics Bertrand Drouvot <[email protected]> 2024-12-18 06:39 [PATCH v11 2/2] per backend I/O statistics Bertrand Drouvot <[email protected]> 2026-05-07 20:32 [PATCH v3 3/3] Better express platform requirements in s_lock.h. Nathan Bossart <[email protected]> 2026-05-07 20:32 [PATCH v5 3/3] Better express platform requirements in s_lock.h. Nathan Bossart <[email protected]> 2026-05-07 20:32 [PATCH v4 3/3] Better express platform requirements in s_lock.h. Nathan Bossart <[email protected]>
This inbox is served by agora; see mirroring instructions for how to clone and mirror all data and code used for this inbox