From: Bertrand Drouvot Date: Mon, 19 Jan 2026 06:27:55 +0000 Subject: [PATCH v3 3/3] Add FLUSH_MIXED support and implement it for RELATION stats This commit extends the non transactional stats infrastructure to support statistics kinds with mixed transaction behavior: some fields are transactional (e.g., tuple inserts/updates/deletes) while others are non transactional (e.g., sequential scans blocks read, ...). It introduces FLUSH_MIXED as a third flush mode type, alongside FLUSH_ANYTIME and FLUSH_AT_TXN_BOUNDARY. For FLUSH_MIXED kinds, a new flush_anytime_cb callback enables partial flushing of only the non transactional fields during running transactions. Some tests are also added. Implementation details: - Add FLUSH_MIXED to PgStat_FlushMode enum - Add flush_anytime_cb to PgStat_KindInfo for partial flushing callback - Update pgstat_flush_pending_entries() to call flush_anytime_cb for FLUSH_MIXED entries when in anytime_only mode - Keep FLUSH_MIXED entries in the pending list after partial flush, as transactional fields still need to be flushed at transaction boundary RELATION stats are making use of FLUSH_MIXED: - Change RELATION from FLUSH_AT_TXN_BOUNDARY to FLUSH_MIXED - Implement pgstat_relation_flush_anytime_cb() to flush only read related stats: numscans, tuples_returned, tuples_fetched, blocks_fetched, blocks_hit - Clear these fields after flushing to prevent double counting when pgstat_relation_flush_cb() runs at transaction commit - Transactional stats (tuples_inserted, tuples_updated, tuples_deleted, live_tuples, dead_tuples) remain pending until transaction boundary The DATABASE kind is also changed from FLUSH_AT_TXN_BOUNDARY to FLUSH_ANYTIME, so that some stats inherited from relations stats are also visible while the transaction is in progress. Remark: We could also imagine adding a new flush_anytime_static_cb() callback for future FLUSH_MIXED fixed amount stats. --- doc/src/sgml/monitoring.sgml | 26 +++++++ src/backend/utils/activity/pgstat.c | 32 ++++++-- src/backend/utils/activity/pgstat_relation.c | 78 ++++++++++++++++++++ src/include/utils/pgstat_internal.h | 9 +++ src/test/isolation/expected/stats.out | 40 ++++++++++ src/test/isolation/expected/stats_1.out | 40 ++++++++++ src/test/isolation/specs/stats.spec | 12 +++ 7 files changed, 229 insertions(+), 8 deletions(-) 12.4% doc/src/sgml/ 47.3% src/backend/utils/activity/ 4.8% src/include/utils/ 31.0% src/test/isolation/expected/ 4.2% src/test/isolation/specs/ diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml index 817fd9f4ca7..94fd2b76136 100644 --- a/doc/src/sgml/monitoring.sgml +++ b/doc/src/sgml/monitoring.sgml @@ -3730,6 +3730,16 @@ description | Waiting for a newly initialized WAL file to reach durable storage + + + All the statistics are updated while the transactions are in progress, except + for xact_commit, xact_rollback, + tup_inserted, tup_updated and + tup_deleted that are updated only when the transactions + finish. + + + @@ -4186,6 +4196,15 @@ description | Waiting for a newly initialized WAL file to reach durable storage + + + The seq_scan, last_seq_scan, + seq_tup_read, idx_scan, + last_idx_scan and idx_tup_fetch + are updated while the transactions are in progress. + + + @@ -4367,6 +4386,13 @@ description | Waiting for a newly initialized WAL file to reach durable storage tuples (see ). + + + The idx_scan, last_idx_scan, + idx_tup_read and idx_tup_fetch + are updated while the transactions are in progress. + + EXPLAIN ANALYZE outputs the total number of index diff --git a/src/backend/utils/activity/pgstat.c b/src/backend/utils/activity/pgstat.c index ab4d9088a9a..6733a739c56 100644 --- a/src/backend/utils/activity/pgstat.c +++ b/src/backend/utils/activity/pgstat.c @@ -287,7 +287,7 @@ static const PgStat_KindInfo pgstat_kind_builtin_infos[PGSTAT_KIND_BUILTIN_SIZE] .fixed_amount = false, .write_to_file = true, - .flush_mode = FLUSH_AT_TXN_BOUNDARY, + .flush_mode = FLUSH_ANYTIME, /* so pg_stat_database entries can be seen in all databases */ .accessed_across_databases = true, @@ -305,7 +305,7 @@ static const PgStat_KindInfo pgstat_kind_builtin_infos[PGSTAT_KIND_BUILTIN_SIZE] .fixed_amount = false, .write_to_file = true, - .flush_mode = FLUSH_AT_TXN_BOUNDARY, + .flush_mode = FLUSH_MIXED, .shared_size = sizeof(PgStatShared_Relation), .shared_data_off = offsetof(PgStatShared_Relation, stats), @@ -313,6 +313,7 @@ static const PgStat_KindInfo pgstat_kind_builtin_infos[PGSTAT_KIND_BUILTIN_SIZE] .pending_size = sizeof(PgStat_TableStatus), .flush_pending_cb = pgstat_relation_flush_cb, + .flush_anytime_cb = pgstat_relation_flush_anytime_cb, .delete_pending_cb = pgstat_relation_delete_pending_cb, .reset_timestamp_cb = pgstat_relation_reset_timestamp_cb, }, @@ -1345,10 +1346,11 @@ pgstat_delete_pending_entry(PgStat_EntryRef *entry_ref) /* * Flush out pending variable-numbered stats. * - * If anytime_only is true, only flushes FLUSH_ANYTIME entries. + * If anytime_only is true, only flushes FLUSH_ANYTIME and FLUSH_MIXED entries, + * using flush_anytime_cb for FLUSH_MIXED. * This is safe to call inside transactions. * - * If anytime_only is false, flushes all entries. + * If anytime_only is false, flushes all entries using flush_pending_cb. */ static bool pgstat_flush_pending_entries(bool nowait, bool anytime_only) @@ -1376,6 +1378,7 @@ pgstat_flush_pending_entries(bool nowait, bool anytime_only) PgStat_Kind kind = key.kind; const PgStat_KindInfo *kind_info = pgstat_get_kind_info(kind); bool did_flush; + bool is_partial_flush = false; dlist_node *next; Assert(!kind_info->fixed_amount); @@ -1395,8 +1398,21 @@ pgstat_flush_pending_entries(bool nowait, bool anytime_only) continue; } - /* flush the stats, if possible */ - did_flush = kind_info->flush_pending_cb(entry_ref, nowait); + /* flush the stats (with the appropriate callback), if possible */ + if (anytime_only && + kind_info->flush_mode == FLUSH_MIXED && + kind_info->flush_anytime_cb != NULL) + { + /* Partial flush of non-transactional fields only */ + did_flush = kind_info->flush_anytime_cb(entry_ref, nowait); + is_partial_flush = true; + } + else + { + /* Full flush */ + did_flush = kind_info->flush_pending_cb(entry_ref, nowait); + is_partial_flush = false; + } Assert(did_flush || nowait); @@ -1406,8 +1422,8 @@ pgstat_flush_pending_entries(bool nowait, bool anytime_only) else next = NULL; - /* if successfully flushed, remove entry */ - if (did_flush) + /* if successfull non partial flush, remove entry */ + if (did_flush && !is_partial_flush) pgstat_delete_pending_entry(entry_ref); else have_pending = true; diff --git a/src/backend/utils/activity/pgstat_relation.c b/src/backend/utils/activity/pgstat_relation.c index feae2ae5f44..d6b799c4354 100644 --- a/src/backend/utils/activity/pgstat_relation.c +++ b/src/backend/utils/activity/pgstat_relation.c @@ -887,6 +887,84 @@ pgstat_relation_flush_cb(PgStat_EntryRef *entry_ref, bool nowait) return true; } +/* + * Flush only non-transactional relation stats. + * + * This is called periodically during running transactions to make some + * statistics visible without waiting for the transaction to finish. + * + * Transactional stats (inserts/updates/deletes and their effects on live/dead + * tuple counts) remain in pending until the transaction ends, at which point + * pgstat_relation_flush_cb() will flush them. + * + * If nowait is true and the lock could not be immediately acquired, returns + * false without flushing the entry. Otherwise returns true. + */ +bool +pgstat_relation_flush_anytime_cb(PgStat_EntryRef *entry_ref, bool nowait) +{ + Oid dboid; + PgStat_TableStatus *lstats; /* pending stats entry */ + PgStatShared_Relation *shtabstats; + PgStat_StatTabEntry *tabentry; /* table entry of shared stats */ + PgStat_StatDBEntry *dbentry; /* pending database entry */ + + dboid = entry_ref->shared_entry->key.dboid; + lstats = (PgStat_TableStatus *) entry_ref->pending; + shtabstats = (PgStatShared_Relation *) entry_ref->shared_stats; + + /* + * Check if there are any non-transactional stats to flush. Avoid + * unnecessarily locking the entry if nothing accumulated. + */ + if (!(lstats->counts.numscans > 0 || + lstats->counts.tuples_returned > 0 || + lstats->counts.tuples_fetched > 0 || + lstats->counts.blocks_fetched > 0 || + lstats->counts.blocks_hit > 0)) + return true; + + if (!pgstat_lock_entry(entry_ref, nowait)) + return false; + + /* Add only the non-transactional values to the shared entry */ + tabentry = &shtabstats->stats; + + tabentry->numscans += lstats->counts.numscans; + if (lstats->counts.numscans) + { + TimestampTz t = GetCurrentTimestamp(); + + if (t > tabentry->lastscan) + tabentry->lastscan = t; + } + tabentry->tuples_returned += lstats->counts.tuples_returned; + tabentry->tuples_fetched += lstats->counts.tuples_fetched; + tabentry->blocks_fetched += lstats->counts.blocks_fetched; + tabentry->blocks_hit += lstats->counts.blocks_hit; + + pgstat_unlock_entry(entry_ref); + + /* Also update the corresponding fields in database stats */ + dbentry = pgstat_prep_database_pending(dboid); + dbentry->tuples_returned += lstats->counts.tuples_returned; + dbentry->tuples_fetched += lstats->counts.tuples_fetched; + dbentry->blocks_fetched += lstats->counts.blocks_fetched; + dbentry->blocks_hit += lstats->counts.blocks_hit; + + /* + * Clear the flushed fields from pending stats to prevent double-counting + * when pgstat_relation_flush_cb() runs at transaction boundary. + */ + lstats->counts.numscans = 0; + lstats->counts.tuples_returned = 0; + lstats->counts.tuples_fetched = 0; + lstats->counts.blocks_fetched = 0; + lstats->counts.blocks_hit = 0; + + return true; +} + void pgstat_relation_delete_pending_cb(PgStat_EntryRef *entry_ref) { diff --git a/src/include/utils/pgstat_internal.h b/src/include/utils/pgstat_internal.h index 46ce90c9624..5f339f6d2ef 100644 --- a/src/include/utils/pgstat_internal.h +++ b/src/include/utils/pgstat_internal.h @@ -236,6 +236,8 @@ typedef enum PgStat_FlushMode * transaction boundary */ FLUSH_ANYTIME, /* All fields can be flushed anytime, * including within transactions */ + FLUSH_MIXED, /* Mix of fields that can be flushed anytime + * or only at transaction boundary */ } PgStat_FlushMode; /* @@ -267,6 +269,12 @@ typedef struct PgStat_KindInfo /* Flush mode */ PgStat_FlushMode flush_mode; + /* + * For FLUSH_MIXED kinds: callback to flush only some fields. If NULL for + * a MIXED kind, treated as FLUSH_AT_TXN_BOUNDARY. + */ + bool (*flush_anytime_cb) (PgStat_EntryRef *entry_ref, bool nowait); + /* * The size of an entry in the shared stats hash table (pointed to by * PgStatShared_HashEntry->body). For fixed-numbered statistics, this is @@ -779,6 +787,7 @@ extern void AtPrepare_PgStat_Relations(PgStat_SubXactStatus *xact_state); extern void PostPrepare_PgStat_Relations(PgStat_SubXactStatus *xact_state); extern bool pgstat_relation_flush_cb(PgStat_EntryRef *entry_ref, bool nowait); +extern bool pgstat_relation_flush_anytime_cb(PgStat_EntryRef *entry_ref, bool nowait); extern void pgstat_relation_delete_pending_cb(PgStat_EntryRef *entry_ref); extern void pgstat_relation_reset_timestamp_cb(PgStatShared_Common *header, TimestampTz ts); diff --git a/src/test/isolation/expected/stats.out b/src/test/isolation/expected/stats.out index cfad309ccf3..6d62b30e4a7 100644 --- a/src/test/isolation/expected/stats.out +++ b/src/test/isolation/expected/stats.out @@ -2245,6 +2245,46 @@ seq_scan|seq_tup_read|n_tup_ins|n_tup_upd|n_tup_del|n_live_tup|n_dead_tup|vacuum (1 row) +starting permutation: s2_begin s2_table_select s1_sleep s1_table_stats s2_table_drop s2_commit +pg_stat_force_next_flush +------------------------ + +(1 row) + +step s2_begin: BEGIN; +step s2_table_select: SELECT * FROM test_stat_tab ORDER BY key, value; +key|value +---+----- +k0 | 1 +(1 row) + +step s1_sleep: SELECT pg_sleep(1.5); +pg_sleep +-------- + +(1 row) + +step s1_table_stats: + SELECT + pg_stat_get_numscans(tso.oid) AS seq_scan, + pg_stat_get_tuples_returned(tso.oid) AS seq_tup_read, + pg_stat_get_tuples_inserted(tso.oid) AS n_tup_ins, + pg_stat_get_tuples_updated(tso.oid) AS n_tup_upd, + pg_stat_get_tuples_deleted(tso.oid) AS n_tup_del, + pg_stat_get_live_tuples(tso.oid) AS n_live_tup, + pg_stat_get_dead_tuples(tso.oid) AS n_dead_tup, + pg_stat_get_vacuum_count(tso.oid) AS vacuum_count + FROM test_stat_oid AS tso + WHERE tso.name = 'test_stat_tab' + +seq_scan|seq_tup_read|n_tup_ins|n_tup_upd|n_tup_del|n_live_tup|n_dead_tup|vacuum_count +--------+------------+---------+---------+---------+----------+----------+------------ + 1| 1| 1| 0| 0| 1| 0| 0 +(1 row) + +step s2_table_drop: DROP TABLE test_stat_tab; +step s2_commit: COMMIT; + starting permutation: s1_track_counts_off s1_table_stats s1_track_counts_on pg_stat_force_next_flush ------------------------ diff --git a/src/test/isolation/expected/stats_1.out b/src/test/isolation/expected/stats_1.out index e1d937784cb..2fade10e817 100644 --- a/src/test/isolation/expected/stats_1.out +++ b/src/test/isolation/expected/stats_1.out @@ -2253,6 +2253,46 @@ seq_scan|seq_tup_read|n_tup_ins|n_tup_upd|n_tup_del|n_live_tup|n_dead_tup|vacuum (1 row) +starting permutation: s2_begin s2_table_select s1_sleep s1_table_stats s2_table_drop s2_commit +pg_stat_force_next_flush +------------------------ + +(1 row) + +step s2_begin: BEGIN; +step s2_table_select: SELECT * FROM test_stat_tab ORDER BY key, value; +key|value +---+----- +k0 | 1 +(1 row) + +step s1_sleep: SELECT pg_sleep(1.5); +pg_sleep +-------- + +(1 row) + +step s1_table_stats: + SELECT + pg_stat_get_numscans(tso.oid) AS seq_scan, + pg_stat_get_tuples_returned(tso.oid) AS seq_tup_read, + pg_stat_get_tuples_inserted(tso.oid) AS n_tup_ins, + pg_stat_get_tuples_updated(tso.oid) AS n_tup_upd, + pg_stat_get_tuples_deleted(tso.oid) AS n_tup_del, + pg_stat_get_live_tuples(tso.oid) AS n_live_tup, + pg_stat_get_dead_tuples(tso.oid) AS n_dead_tup, + pg_stat_get_vacuum_count(tso.oid) AS vacuum_count + FROM test_stat_oid AS tso + WHERE tso.name = 'test_stat_tab' + +seq_scan|seq_tup_read|n_tup_ins|n_tup_upd|n_tup_del|n_live_tup|n_dead_tup|vacuum_count +--------+------------+---------+---------+---------+----------+----------+------------ + 0| 0| 1| 0| 0| 1| 0| 0 +(1 row) + +step s2_table_drop: DROP TABLE test_stat_tab; +step s2_commit: COMMIT; + starting permutation: s1_track_counts_off s1_table_stats s1_track_counts_on pg_stat_force_next_flush ------------------------ diff --git a/src/test/isolation/specs/stats.spec b/src/test/isolation/specs/stats.spec index da16710da0f..1b0168e6176 100644 --- a/src/test/isolation/specs/stats.spec +++ b/src/test/isolation/specs/stats.spec @@ -50,6 +50,8 @@ step s1_rollback { ROLLBACK; } step s1_prepare_a { PREPARE TRANSACTION 'a'; } step s1_commit_prepared_a { COMMIT PREPARED 'a'; } step s1_rollback_prepared_a { ROLLBACK PREPARED 'a'; } +# Has to be greater than PGSTAT_ANYTIME_FLUSH_INTERVAL +step s1_sleep { SELECT pg_sleep(1.5); } # Function stats steps step s1_ff { SELECT pg_stat_force_next_flush(); } @@ -138,6 +140,7 @@ step s2_commit { COMMIT; } step s2_commit_prepared_a { COMMIT PREPARED 'a'; } step s2_rollback_prepared_a { ROLLBACK PREPARED 'a'; } step s2_ff { SELECT pg_stat_force_next_flush(); } +step s2_table_drop { DROP TABLE test_stat_tab; } # Function stats steps step s2_track_funcs_all { SET track_functions = 'all'; } @@ -435,6 +438,15 @@ permutation s1_table_drop s1_table_stats +### Check that some stats are updated (seq_scan and seq_tup_read) +### while the transaction is still running +permutation + s2_begin + s2_table_select + s1_sleep + s1_table_stats + s2_table_drop + s2_commit ### Check that we don't count changes with track counts off, but allow access ### to prior stats -- 2.34.1 --lCc1MWfC7i2uqs8g--