public inbox for [email protected]
help / color / mirror / Atom feed[PATCH v4 1/3] Adding per backend commit and rollback counters
5+ messages / 4 participants
[nested] [flat]
* [PATCH v4 1/3] Adding per backend commit and rollback counters
@ 2025-08-04 08:14 Bertrand Drouvot <[email protected]>
0 siblings, 0 replies; 5+ messages in thread
From: Bertrand Drouvot @ 2025-08-04 08:14 UTC (permalink / raw)
It relies on the existing per backend statistics that has been added in
9aea73fc61d. The new pending counters are updated when the database ones are
flushed (to reduce the overhead of incrementing new counters).
---
src/backend/utils/activity/pgstat_backend.c | 42 +++++++++++++++++++-
src/backend/utils/activity/pgstat_database.c | 7 ++++
src/include/pgstat.h | 15 +++++++
src/include/utils/pgstat_internal.h | 3 +-
4 files changed, 65 insertions(+), 2 deletions(-)
74.4% src/backend/utils/activity/
7.8% src/include/utils/
17.7% src/include/
diff --git a/src/backend/utils/activity/pgstat_backend.c b/src/backend/utils/activity/pgstat_backend.c
index 8714a85e2d9..47ce61e5093 100644
--- a/src/backend/utils/activity/pgstat_backend.c
+++ b/src/backend/utils/activity/pgstat_backend.c
@@ -36,7 +36,7 @@
* reported within critical sections so we use static memory in order to avoid
* memory allocation.
*/
-static PgStat_BackendPending PendingBackendStats;
+PgStat_BackendPending PendingBackendStats;
static bool backend_has_iostats = false;
/*
@@ -47,6 +47,11 @@ static bool backend_has_iostats = false;
*/
static WalUsage prevBackendWalUsage;
+/*
+ * For backend commit and rollback statistics.
+ */
+bool backend_has_xactstats = false;
+
/*
* Utility routines to report I/O stats for backends, kept here to avoid
* exposing PendingBackendStats to the outside world.
@@ -259,6 +264,34 @@ pgstat_flush_backend_entry_wal(PgStat_EntryRef *entry_ref)
prevBackendWalUsage = pgWalUsage;
}
+/*
+ * Flush out locally pending backend transaction statistics. Locking is managed
+ * by the caller.
+ */
+static void
+pgstat_flush_backend_entry_xact(PgStat_EntryRef *entry_ref)
+{
+ PgStatShared_Backend *shbackendent;
+
+ /*
+ * This function can be called even if nothing at all has happened for
+ * transaction statistics. In this case, avoid unnecessarily modifying
+ * the stats entry.
+ */
+ if (!backend_has_xactstats)
+ return;
+
+ shbackendent = (PgStatShared_Backend *) entry_ref->shared_stats;
+
+ shbackendent->stats.xact_commit += PendingBackendStats.pending_xact_commit;
+ shbackendent->stats.xact_rollback += PendingBackendStats.pending_xact_rollback;
+
+ PendingBackendStats.pending_xact_commit = 0;
+ PendingBackendStats.pending_xact_rollback = 0;
+
+ backend_has_xactstats = false;
+}
+
/*
* Flush out locally pending backend statistics
*
@@ -283,6 +316,10 @@ pgstat_flush_backend(bool nowait, bits32 flags)
pgstat_backend_wal_have_pending())
has_pending_data = true;
+ /* Some transaction data pending? */
+ if ((flags & PGSTAT_BACKEND_FLUSH_XACT) && backend_has_xactstats)
+ has_pending_data = true;
+
if (!has_pending_data)
return false;
@@ -298,6 +335,9 @@ pgstat_flush_backend(bool nowait, bits32 flags)
if (flags & PGSTAT_BACKEND_FLUSH_WAL)
pgstat_flush_backend_entry_wal(entry_ref);
+ if (flags & PGSTAT_BACKEND_FLUSH_XACT)
+ pgstat_flush_backend_entry_xact(entry_ref);
+
pgstat_unlock_entry(entry_ref);
return false;
diff --git a/src/backend/utils/activity/pgstat_database.c b/src/backend/utils/activity/pgstat_database.c
index b31f20d41bc..400a8c5d734 100644
--- a/src/backend/utils/activity/pgstat_database.c
+++ b/src/backend/utils/activity/pgstat_database.c
@@ -342,6 +342,13 @@ pgstat_update_dbstats(TimestampTz ts)
dbentry->blk_read_time += pgStatBlockReadTime;
dbentry->blk_write_time += pgStatBlockWriteTime;
+ /* Do the same for backend stats */
+ PendingBackendStats.pending_xact_commit += pgStatXactCommit;
+ PendingBackendStats.pending_xact_rollback += pgStatXactRollback;
+
+ backend_has_xactstats = true;
+ pgstat_report_fixed = true;
+
if (pgstat_should_report_connstat())
{
long secs;
diff --git a/src/include/pgstat.h b/src/include/pgstat.h
index 202bd2d5ace..0fdbaf79780 100644
--- a/src/include/pgstat.h
+++ b/src/include/pgstat.h
@@ -490,6 +490,8 @@ typedef struct PgStat_Backend
TimestampTz stat_reset_timestamp;
PgStat_BktypeIO io_stats;
PgStat_WalCounters wal_counters;
+ PgStat_Counter xact_commit;
+ PgStat_Counter xact_rollback;
} PgStat_Backend;
/* ---------
@@ -502,6 +504,12 @@ typedef struct PgStat_BackendPending
* Backend statistics store the same amount of IO data as PGSTAT_KIND_IO.
*/
PgStat_PendingIO pending_io;
+
+ /*
+ * Transaction statistics pending flush.
+ */
+ PgStat_Counter pending_xact_commit;
+ PgStat_Counter pending_xact_rollback;
} PgStat_BackendPending;
/*
@@ -801,6 +809,13 @@ extern PGDLLIMPORT int pgstat_track_functions;
extern PGDLLIMPORT int pgstat_fetch_consistency;
+/*
+ * Variables in pgstat_backend.c
+ */
+
+extern PGDLLIMPORT PgStat_BackendPending PendingBackendStats;
+extern PGDLLIMPORT bool backend_has_xactstats;
+
/*
* Variables in pgstat_bgwriter.c
*/
diff --git a/src/include/utils/pgstat_internal.h b/src/include/utils/pgstat_internal.h
index 6cf00008f63..b97184c6539 100644
--- a/src/include/utils/pgstat_internal.h
+++ b/src/include/utils/pgstat_internal.h
@@ -616,7 +616,8 @@ extern void pgstat_archiver_snapshot_cb(void);
/* flags for pgstat_flush_backend() */
#define PGSTAT_BACKEND_FLUSH_IO (1 << 0) /* Flush I/O statistics */
#define PGSTAT_BACKEND_FLUSH_WAL (1 << 1) /* Flush WAL statistics */
-#define PGSTAT_BACKEND_FLUSH_ALL (PGSTAT_BACKEND_FLUSH_IO | PGSTAT_BACKEND_FLUSH_WAL)
+#define PGSTAT_BACKEND_FLUSH_XACT (1 << 2) /* Flush xact statistics */
+#define PGSTAT_BACKEND_FLUSH_ALL (PGSTAT_BACKEND_FLUSH_IO | PGSTAT_BACKEND_FLUSH_WAL | PGSTAT_BACKEND_FLUSH_XACT)
extern bool pgstat_flush_backend(bool nowait, bits32 flags);
extern bool pgstat_backend_flush_cb(bool nowait);
--
2.34.1
--jg9kqX3osLGO+8A7
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v4-0002-Adding-XID-generation-count-per-backend.patch"
^ permalink raw reply [nested|flat] 5+ messages in thread
* Re:Re: Fix logical decoding not track transaction during SNAPBUILD_BUILDING_SNAPSHOT
@ 2026-01-28 19:06 ocean_li_996 <[email protected]>
2026-01-29 03:33 ` Re: Re: Fix logical decoding not track transaction during SNAPBUILD_BUILDING_SNAPSHOT Ajin Cherian <[email protected]>
0 siblings, 1 reply; 5+ messages in thread
From: ocean_li_996 @ 2026-01-28 19:06 UTC (permalink / raw)
To: Ajin Cherian <[email protected]>; +Cc: [email protected] <[email protected]>
Hi Ajin,
At 2026-01-28 11:32:41, "Ajin Cherian" <[email protected]> wrote:
>I agree with your analysis and approach, but when I tried out the
>patch (applying patch 0002 for the tests and patch 0004), I see the
>tests in contrib/test_decoding failing.
>Similarly, applying patch 0002 and 0003 also results in the tests
>failing. So, I am not sure how your minimal fix fixes the problem. Am
>I doing something wrong?
>Does patch 0003 and 0004 have to be applied on top of 0001? That
>doesn't seem to be the case, as both make the same code change and
>don't apply cleanly.
0002 patch is only a test case. And 0001, 0003 and 0004 are independt fix patch.
I appied 0002 + 0003 and 0002 + 0004 separately in master. And both the tests in
contrib/test_decoding were passed. Can you provide more details about the failed
tests(such as which tests and the diff between expected and ressult).
Regards
Haiyang Li
^ permalink raw reply [nested|flat] 5+ messages in thread
* Re: Re: Fix logical decoding not track transaction during SNAPBUILD_BUILDING_SNAPSHOT
2026-01-28 19:06 Re:Re: Fix logical decoding not track transaction during SNAPBUILD_BUILDING_SNAPSHOT ocean_li_996 <[email protected]>
@ 2026-01-29 03:33 ` Ajin Cherian <[email protected]>
2026-01-29 06:29 ` Re: Fix logical decoding not track transaction during SNAPBUILD_BUILDING_SNAPSHOT =?utf-8?B?Y2NhNTUwNw==?= <[email protected]>
0 siblings, 1 reply; 5+ messages in thread
From: Ajin Cherian @ 2026-01-29 03:33 UTC (permalink / raw)
To: ocean_li_996 <[email protected]>; +Cc: [email protected] <[email protected]>
On Thu, Jan 29, 2026 at 6:07 AM ocean_li_996 <[email protected]> wrote:
>
> Hi Ajin,
>
> At 2026-01-28 11:32:41, "Ajin Cherian" <[email protected]> wrote:
> >I agree with your analysis and approach, but when I tried out the
> >patch (applying patch 0002 for the tests and patch 0004), I see the
> >tests in contrib/test_decoding failing.
> >Similarly, applying patch 0002 and 0003 also results in the tests
> >failing. So, I am not sure how your minimal fix fixes the problem. Am
> >I doing something wrong?
> >Does patch 0003 and 0004 have to be applied on top of 0001? That
> >doesn't seem to be the case, as both make the same code change and
> >don't apply cleanly.
>
> 0002 patch is only a test case. And 0001, 0003 and 0004 are independt fix patch.
>
> I appied 0002 + 0003 and 0002 + 0004 separately in master. And both the tests in
>
> contrib/test_decoding were passed. Can you provide more details about the failed
>
> tests(such as which tests and the diff between expected and ressult).
Hi Haiyang,
I tested with patch 0002+0004 on HEAD, and the test added by patch
0002 is failing like below.
not ok 15 - snapshot_build 289 ms
# (test process exited with exit code 1)
I see the postgres crashed and when I look at the core file, I see the
below stack trace:
Core was generated by
`/home/ajin/postgresql/postgres3/postgres/tmp_install/home/ajin/install-oss/bin/postgres
'' '' '' '' '' '' '''.
Program terminated with signal SIGABRT, Aborted.
#0 0x00007436092c7e9c in __pthread_kill_implementation () from /lib64/libc.so.6
Missing rpms, try: dnf --enablerepo='*debug*' install
zlib-ng-compat-debuginfo-2.2.3-2.el10.x86_64
glibc-debuginfo-2.39-38.el10.x86_64
libicu-debuginfo-74.2-5.el10.x86_64
libstdc++-debuginfo-14.3.1-2.1.el10.x86_64
libgcc-debuginfo-14.3.1-2.1.el10.x86_64
(gdb) bt
#0 0x00007436092c7e9c in __pthread_kill_implementation () from /lib64/libc.so.6
#1 0x0000743609271a96 in raise () from /lib64/libc.so.6
#2 0x00007436092598fa in abort () from /lib64/libc.so.6
#3 0x0000000000a00a93 in ExceptionalCondition
(conditionName=conditionName@entry=0xb1e039 "txn->ninvalidations ==
0", fileName=fileName@entry=0xb1dcd8 "reorderbuffer.c",
lineNumber=lineNumber@entry=3207) at assert.c:65
#4 0x0000000000836c94 in ReorderBufferForget (rb=0x3cf96a40,
xid=xid@entry=1136, lsn=34819568) at reorderbuffer.c:3207
#5 0x0000000000823aea in DecodeCommit (ctx=ctx@entry=0x3cf869d0,
buf=buf@entry=0x7ffd069389a0, parsed=parsed@entry=0x7ffd069387d0,
xid=xid@entry=1136, two_phase=false) at decode.c:707
#6 0x000000000082497c in xact_decode (ctx=ctx@entry=0x3cf869d0,
buf=buf@entry=0x7ffd069389a0) at decode.c:237
#7 0x00000000008246eb in LogicalDecodingProcessRecord
(ctx=ctx@entry=0x3cf869d0, record=0x3cf86da8) at decode.c:116
#8 0x0000000000829602 in DecodingContextFindStartpoint
(ctx=ctx@entry=0x3cf869d0) at logical.c:647
#9 0x000000000084e8ea in create_logical_replication_slot
(name=name@entry=0x3ce8fb70 "isolation_slot",
plugin=plugin@entry=0x3ce8fc10 "test_decoding",
temporary=temporary@entry=false, two_phase=two_phase@entry=false,
failover=failover@entry=false,
restart_lsn=restart_lsn@entry=0, find_startpoint=true) at slotfuncs.c:177
#10 0x000000000084f300 in pg_create_logical_replication_slot
(fcinfo=<optimized out>) at slotfuncs.c:207
#11 0x00000000006c77c4 in ExecMakeTableFunctionResult
(setexpr=0x3cf61b18, econtext=0x3cf61968, argContext=<optimized out>,
expectedDesc=0x3cf79f48, randomAccess=false) at execSRF.c:234
#12 0x00000000006da839 in FunctionNext (node=node@entry=0x3cf61758) at
nodeFunctionscan.c:94
(gdb) f 4
#4 0x0000000000836c94 in ReorderBufferForget (rb=0x3cf96a40,
xid=xid@entry=1136, lsn=34819568) at reorderbuffer.c:3207
3207 Assert(txn->ninvalidations == 0);
(gdb) p *txn
$1 = {txn_flags = 1, xid = 1136, toplevel_xid = 0, gid = 0x0,
first_lsn = 34814328, final_lsn = 34819568, end_lsn = 0, toptxn = 0x0,
restart_decoding_lsn = 0, origin_id = 0, origin_lsn = 0, {commit_time
= 0, prepare_time = 0, abort_time = 0}, base_snapshot = 0x0,
base_snapshot_lsn = 0, base_snapshot_node = {prev = 0x0, next =
0x0}, snapshot_now = 0x0, command_id = 4294967295, nentries = 14,
nentries_mem = 14, changes = {head = {prev = 0x3cfb5510, next =
0x3cfb4ae8}}, tuplecids = {head = {prev = 0x3cfb5440, next =
0x3cfb4a80}},
ntuplecids = 13, tuplecid_hash = 0x0, toast_hash = 0x0, subtxns =
{head = {prev = 0x3cfb2c68, next = 0x3cfb2c68}}, nsubtxns = 0,
ninvalidations = 22, invalidations = 0x3cf96c50,
ninvalidations_distributed = 0, invalidations_distributed = 0x0, node
= {prev = 0x3cfb2b30,
next = 0x3cf96a48}, catchange_node = {prev = 0x3cf96a68, next =
0x3cf96a68}, txn_node = {first_child = 0x0, next_sibling = 0x0,
prev_or_parent = 0x0}, size = 1472, total_size = 1472,
output_plugin_private = 0x0}
Looks like the assert in ReorderBufferForget failed because
ninvalidations is not 0.
You need to build with asserts enabled.
regards,
Ajin Cherian
Fujtitsu Australia
^ permalink raw reply [nested|flat] 5+ messages in thread
* Re: Fix logical decoding not track transaction during SNAPBUILD_BUILDING_SNAPSHOT
2026-01-28 19:06 Re:Re: Fix logical decoding not track transaction during SNAPBUILD_BUILDING_SNAPSHOT ocean_li_996 <[email protected]>
2026-01-29 03:33 ` Re: Re: Fix logical decoding not track transaction during SNAPBUILD_BUILDING_SNAPSHOT Ajin Cherian <[email protected]>
@ 2026-01-29 06:29 ` =?utf-8?B?Y2NhNTUwNw==?= <[email protected]>
2026-01-29 11:13 ` Re: Fix logical decoding not track transaction during SNAPBUILD_BUILDING_SNAPSHOT Ajin Cherian <[email protected]>
0 siblings, 1 reply; 5+ messages in thread
From: =?utf-8?B?Y2NhNTUwNw==?= @ 2026-01-29 06:29 UTC (permalink / raw)
To: =?utf-8?B?QWppbiBDaGVyaWFu?= <[email protected]>; =?utf-8?B?b2NlYW5fbGlfOTk2?= <[email protected]>; +Cc: =?utf-8?B?cGdzcWwtaGFja2Vyc0BsaXN0cy5wb3N0Z3Jlc3FsLm9yZw==?= <[email protected]>
> Looks like the assert in ReorderBufferForget failed because
> ninvalidations is not 0.
I think it can be fixed by this:
```
@@ -282,18 +286,24 @@ xact_decode(LogicalDecodingContext *ctx, XLogRecordBuffer *buf)
{
TransactionId xid;
xl_xact_invals *invals;
+ bool has_snapshot;
xid = XLogRecGetXid(r);
invals = (xl_xact_invals *) XLogRecGetData(r);
+ has_snapshot =
+ SnapBuildCurrentState(builder) >= SNAPBUILD_FULL_SNAPSHOT;
/*
* Execute the invalidations for xid-less transactions,
* otherwise, accumulate them so that they can be processed at
* the commit time.
+ *
+ * Note that we only need to do this when we are not fast-forwarding
+ * and there is a snapshot.
*/
if (TransactionIdIsValid(xid))
{
- if (!ctx->fast_forward)
+ if (!ctx->fast_forward && has_snapshot)
ReorderBufferAddInvalidations(reorder, xid,
buf->origptr,
invals->nmsgs,
@@ -301,7 +311,7 @@ xact_decode(LogicalDecodingContext *ctx, XLogRecordBuffer *buf)
ReorderBufferXidSetCatalogChanges(ctx->reorder, xid,
buf->origptr);
}
- else if (!ctx->fast_forward)
+ else if (!ctx->fast_forward && has_snapshot)
ReorderBufferImmediateInvalidation(ctx->reorder,
invals->nmsgs,
invals->msgs);
```
--
Regards,
ChangAo Chen
^ permalink raw reply [nested|flat] 5+ messages in thread
* Re: Fix logical decoding not track transaction during SNAPBUILD_BUILDING_SNAPSHOT
2026-01-28 19:06 Re:Re: Fix logical decoding not track transaction during SNAPBUILD_BUILDING_SNAPSHOT ocean_li_996 <[email protected]>
2026-01-29 03:33 ` Re: Re: Fix logical decoding not track transaction during SNAPBUILD_BUILDING_SNAPSHOT Ajin Cherian <[email protected]>
2026-01-29 06:29 ` Re: Fix logical decoding not track transaction during SNAPBUILD_BUILDING_SNAPSHOT =?utf-8?B?Y2NhNTUwNw==?= <[email protected]>
@ 2026-01-29 11:13 ` Ajin Cherian <[email protected]>
0 siblings, 0 replies; 5+ messages in thread
From: Ajin Cherian @ 2026-01-29 11:13 UTC (permalink / raw)
To: cca5507 <[email protected]>; +Cc: ocean_li_996 <[email protected]>; [email protected] <[email protected]>
On Thu, Jan 29, 2026 at 5:29 PM cca5507 <[email protected]> wrote:
>
> > Looks like the assert in ReorderBufferForget failed because
> > ninvalidations is not 0.
>
> I think it can be fixed by this:
>
> ```
> @@ -282,18 +286,24 @@ xact_decode(LogicalDecodingContext *ctx, XLogRecordBuffer *buf)
> {
> TransactionId xid;
> xl_xact_invals *invals;
> + bool has_snapshot;
>
> xid = XLogRecGetXid(r);
> invals = (xl_xact_invals *) XLogRecGetData(r);
> + has_snapshot =
> + SnapBuildCurrentState(builder) >= SNAPBUILD_FULL_SNAPSHOT;
>
> /*
> * Execute the invalidations for xid-less transactions,
> * otherwise, accumulate them so that they can be processed at
> * the commit time.
> + *
> + * Note that we only need to do this when we are not fast-forwarding
> + * and there is a snapshot.
> */
> if (TransactionIdIsValid(xid))
> {
> - if (!ctx->fast_forward)
> + if (!ctx->fast_forward && has_snapshot)
> ReorderBufferAddInvalidations(reorder, xid,
> buf->origptr,
> invals->nmsgs,
> @@ -301,7 +311,7 @@ xact_decode(LogicalDecodingContext *ctx, XLogRecordBuffer *buf)
> ReorderBufferXidSetCatalogChanges(ctx->reorder, xid,
> buf->origptr);
> }
> - else if (!ctx->fast_forward)
> + else if (!ctx->fast_forward && has_snapshot)
> ReorderBufferImmediateInvalidation(ctx->reorder,
> invals->nmsgs,
> invals->msgs);
> ```
>
> --
Yes, this works.
regards,
Ajin Cherian
Fujitsu Australia
^ permalink raw reply [nested|flat] 5+ messages in thread
end of thread, other threads:[~2026-01-29 11:13 UTC | newest]
Thread overview: 5+ messages (download: mbox mbox.gz follow: Atom feed)
-- links below jump to the message on this page --
2025-08-04 08:14 [PATCH v4 1/3] Adding per backend commit and rollback counters Bertrand Drouvot <[email protected]>
2026-01-28 19:06 Re:Re: Fix logical decoding not track transaction during SNAPBUILD_BUILDING_SNAPSHOT ocean_li_996 <[email protected]>
2026-01-29 03:33 ` Re: Re: Fix logical decoding not track transaction during SNAPBUILD_BUILDING_SNAPSHOT Ajin Cherian <[email protected]>
2026-01-29 06:29 ` Re: Fix logical decoding not track transaction during SNAPBUILD_BUILDING_SNAPSHOT =?utf-8?B?Y2NhNTUwNw==?= <[email protected]>
2026-01-29 11:13 ` Re: Fix logical decoding not track transaction during SNAPBUILD_BUILDING_SNAPSHOT Ajin Cherian <[email protected]>
This inbox is served by agora; see mirroring instructions
for how to clone and mirror all data and code used for this inbox