public inbox for [email protected]help / color / mirror / Atom feed
[PATCH v8 1/8] nodeAgg: separate context for each hashtable 113+ messages / 14 participants [nested] [flat]
* [PATCH v8 1/8] nodeAgg: separate context for each hashtable @ 2020-03-20 04:03 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 113+ messages in thread From: Justin Pryzby @ 2020-03-20 04:03 UTC (permalink / raw) --- src/backend/executor/execExpr.c | 2 +- src/backend/executor/nodeAgg.c | 82 +++++++++++++++++++-------------- src/include/executor/nodeAgg.h | 2 + src/include/nodes/execnodes.h | 2 - 4 files changed, 51 insertions(+), 37 deletions(-) diff --git a/src/backend/executor/execExpr.c b/src/backend/executor/execExpr.c index 1370ffec50..039c5a8b5f 100644 --- a/src/backend/executor/execExpr.c +++ b/src/backend/executor/execExpr.c @@ -3241,7 +3241,7 @@ ExecBuildAggTransCall(ExprState *state, AggState *aggstate, int adjust_jumpnull = -1; if (ishash) - aggcontext = aggstate->hashcontext; + aggcontext = aggstate->perhash[setno].hashcontext; else aggcontext = aggstate->aggcontexts[setno]; diff --git a/src/backend/executor/nodeAgg.c b/src/backend/executor/nodeAgg.c index 44c159ab2a..1d319f49d0 100644 --- a/src/backend/executor/nodeAgg.c +++ b/src/backend/executor/nodeAgg.c @@ -191,8 +191,7 @@ * So we create an array, aggcontexts, with an ExprContext for each grouping * set in the largest rollup that we're going to process, and use the * per-tuple memory context of those ExprContexts to store the aggregate - * transition values. hashcontext is the single context created to support - * all hash tables. + * transition values. * * Spilling To Disk * @@ -457,7 +456,7 @@ select_current_set(AggState *aggstate, int setno, bool is_hash) * ExecAggPlainTransByRef(). */ if (is_hash) - aggstate->curaggcontext = aggstate->hashcontext; + aggstate->curaggcontext = aggstate->perhash[setno].hashcontext; else aggstate->curaggcontext = aggstate->aggcontexts[setno]; @@ -1424,8 +1423,7 @@ find_unaggregated_cols_walker(Node *node, Bitmapset **colnos) * grouping set for which we're doing hashing. * * The contents of the hash tables always live in the hashcontext's per-tuple - * memory context (there is only one of these for all tables together, since - * they are all reset at the same time). + * memory context. */ static void build_hash_tables(AggState *aggstate) @@ -1465,8 +1463,8 @@ static void build_hash_table(AggState *aggstate, int setno, long nbuckets) { AggStatePerHash perhash = &aggstate->perhash[setno]; - MemoryContext metacxt = aggstate->hash_metacxt; - MemoryContext hashcxt = aggstate->hashcontext->ecxt_per_tuple_memory; + MemoryContext metacxt = perhash->hash_metacxt; + MemoryContext hashcxt = perhash->hashcontext->ecxt_per_tuple_memory; MemoryContext tmpcxt = aggstate->tmpcontext->ecxt_per_tuple_memory; Size additionalsize; @@ -1776,10 +1774,15 @@ static void hash_agg_check_limits(AggState *aggstate) { uint64 ngroups = aggstate->hash_ngroups_current; - Size meta_mem = MemoryContextMemAllocated( - aggstate->hash_metacxt, true); - Size hash_mem = MemoryContextMemAllocated( - aggstate->hashcontext->ecxt_per_tuple_memory, true); + Size meta_mem = 0; + Size hash_mem = 0; + + for (int i = 0; i < aggstate->num_hashes; ++i) + meta_mem = MemoryContextMemAllocated( + aggstate->perhash[i].hash_metacxt, true); + for (int i = 0; i < aggstate->num_hashes; ++i) + hash_mem += MemoryContextMemAllocated( + aggstate->perhash[i].hashcontext->ecxt_per_tuple_memory, true); /* * Don't spill unless there's at least one group in the hash table so we @@ -1837,8 +1840,8 @@ hash_agg_enter_spill_mode(AggState *aggstate) static void hash_agg_update_metrics(AggState *aggstate, bool from_tape, int npartitions) { - Size meta_mem; - Size hash_mem; + Size meta_mem = 0; + Size hash_mem = 0; Size buffer_mem; Size total_mem; @@ -1846,12 +1849,16 @@ hash_agg_update_metrics(AggState *aggstate, bool from_tape, int npartitions) aggstate->aggstrategy != AGG_HASHED) return; - /* memory for the hash table itself */ - meta_mem = MemoryContextMemAllocated(aggstate->hash_metacxt, true); - /* memory for the group keys and transition states */ - hash_mem = MemoryContextMemAllocated( - aggstate->hashcontext->ecxt_per_tuple_memory, true); + for (int i = 0; i < aggstate->num_hashes; ++i) + { + /* memory for the hash table itself */ + meta_mem += MemoryContextMemAllocated( + aggstate->perhash[i].hash_metacxt, true); + /* memory for the group keys and transition states */ + hash_mem += MemoryContextMemAllocated( + aggstate->perhash[i].hashcontext->ecxt_per_tuple_memory, true); + } /* memory for read/write tape buffers, if spilled */ buffer_mem = npartitions * HASHAGG_WRITE_BUFFER_SIZE; @@ -2557,9 +2564,11 @@ agg_refill_hash_table(AggState *aggstate) aggstate->all_pergroups[setoff] = NULL; /* free memory and reset hash tables */ - ReScanExprContext(aggstate->hashcontext); for (int setno = 0; setno < aggstate->num_hashes; setno++) + { + ReScanExprContext(aggstate->perhash[setno].hashcontext); ResetTupleHashTable(aggstate->perhash[setno].hashtable); + } aggstate->hash_ngroups_current = 0; @@ -3217,6 +3226,10 @@ ExecInitAgg(Agg *node, EState *estate, int eflags) aggstate->aggcontexts = (ExprContext **) palloc0(sizeof(ExprContext *) * numGroupingSets); + aggstate->num_hashes = numHashes; + if (numHashes) + aggstate->perhash = palloc0(sizeof(AggStatePerHashData) * numHashes); + /* * Create expression contexts. We need three or more, one for * per-input-tuple processing, one for per-output-tuple processing, one @@ -3240,10 +3253,10 @@ ExecInitAgg(Agg *node, EState *estate, int eflags) aggstate->aggcontexts[i] = aggstate->ss.ps.ps_ExprContext; } - if (use_hashing) + for (i = 0 ; i < numHashes; ++i) { ExecAssignExprContext(estate, &aggstate->ss.ps); - aggstate->hashcontext = aggstate->ss.ps.ps_ExprContext; + aggstate->perhash[i].hashcontext = aggstate->ss.ps.ps_ExprContext; } ExecAssignExprContext(estate, &aggstate->ss.ps); @@ -3332,11 +3345,8 @@ ExecInitAgg(Agg *node, EState *estate, int eflags) * compare functions. Accumulate all_grouped_cols in passing. */ aggstate->phases = palloc0(numPhases * sizeof(AggStatePerPhaseData)); - - aggstate->num_hashes = numHashes; if (numHashes) { - aggstate->perhash = palloc0(sizeof(AggStatePerHashData) * numHashes); aggstate->phases[0].numsets = 0; aggstate->phases[0].gset_lengths = palloc(numHashes * sizeof(int)); aggstate->phases[0].grouped_cols = palloc(numHashes * sizeof(Bitmapset *)); @@ -3528,10 +3538,12 @@ ExecInitAgg(Agg *node, EState *estate, int eflags) uint64 totalGroups = 0; int i; - aggstate->hash_metacxt = AllocSetContextCreate( - aggstate->ss.ps.state->es_query_cxt, - "HashAgg meta context", - ALLOCSET_DEFAULT_SIZES); + for (i = 0; i < aggstate->num_hashes; i++) + aggstate->perhash[i].hash_metacxt = AllocSetContextCreate( + aggstate->ss.ps.state->es_query_cxt, + "HashAgg meta context", + ALLOCSET_DEFAULT_SIZES); + aggstate->hash_spill_slot = ExecInitExtraTupleSlot( estate, scanDesc, &TTSOpsMinimalTuple); @@ -4461,10 +4473,11 @@ ExecEndAgg(AggState *node) hashagg_reset_spill_state(node); - if (node->hash_metacxt != NULL) + for (setno = 0; setno < node->num_hashes; setno++) { - MemoryContextDelete(node->hash_metacxt); - node->hash_metacxt = NULL; + MemoryContext *metacxt = &node->perhash[setno].hash_metacxt; + MemoryContextDelete(*metacxt); + *metacxt = NULL; } for (transno = 0; transno < node->numtrans; transno++) @@ -4481,8 +4494,8 @@ ExecEndAgg(AggState *node) /* And ensure any agg shutdown callbacks have been called */ for (setno = 0; setno < numGroupingSets; setno++) ReScanExprContext(node->aggcontexts[setno]); - if (node->hashcontext) - ReScanExprContext(node->hashcontext); + for (setno = 0; setno < node->num_hashes; setno++) + ReScanExprContext(node->perhash[setno].hashcontext); /* * We don't actually free any ExprContexts here (see comment in @@ -4591,7 +4604,8 @@ ExecReScanAgg(AggState *node) node->hash_spill_mode = false; node->hash_ngroups_current = 0; - ReScanExprContext(node->hashcontext); + for (setno = 0; setno < node->num_hashes; setno++) + ReScanExprContext(node->perhash[setno].hashcontext); /* Rebuild an empty hash table */ build_hash_tables(node); node->table_filled = false; diff --git a/src/include/executor/nodeAgg.h b/src/include/executor/nodeAgg.h index a5b8a004d1..247bb65bd6 100644 --- a/src/include/executor/nodeAgg.h +++ b/src/include/executor/nodeAgg.h @@ -307,6 +307,8 @@ typedef struct AggStatePerHashData AttrNumber *hashGrpColIdxInput; /* hash col indices in input slot */ AttrNumber *hashGrpColIdxHash; /* indices in hash table tuples */ Agg *aggnode; /* original Agg node, for numGroups etc. */ + MemoryContext hash_metacxt; /* memory for hash table itself */ + ExprContext *hashcontext; /* context for hash table data */ } AggStatePerHashData; diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h index 3d27d50f09..ddf0b43916 100644 --- a/src/include/nodes/execnodes.h +++ b/src/include/nodes/execnodes.h @@ -2051,7 +2051,6 @@ typedef struct AggState int current_phase; /* current phase number */ AggStatePerAgg peragg; /* per-Aggref information */ AggStatePerTrans pertrans; /* per-Trans state information */ - ExprContext *hashcontext; /* econtexts for long-lived data (hashtable) */ ExprContext **aggcontexts; /* econtexts for long-lived data (per GS) */ ExprContext *tmpcontext; /* econtext for input expressions */ #define FIELDNO_AGGSTATE_CURAGGCONTEXT 14 @@ -2079,7 +2078,6 @@ typedef struct AggState /* these fields are used in AGG_HASHED and AGG_MIXED modes: */ bool table_filled; /* hash table filled yet? */ int num_hashes; - MemoryContext hash_metacxt; /* memory for hash table itself */ struct HashTapeInfo *hash_tapeinfo; /* metadata for spill tapes */ struct HashAggSpill *hash_spills; /* HashAggSpill for each grouping set, exists only during first pass */ -- 2.17.0 --ZljC5FVPx7rxDQQ8 Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v8-0002-explain-to-show-tuplehash-bucket-and-memory-stats.patch" ^ permalink raw reply [nested|flat] 113+ messages in thread
* Failed transaction statistics to measure the logical replication progress @ 2021-07-08 06:54 [email protected] <[email protected]> 0 siblings, 3 replies; 113+ messages in thread From: [email protected] @ 2021-07-08 06:54 UTC (permalink / raw) To: [email protected] <[email protected]> Hello, hackers When the current HEAD fails during logical decoding, the failure increments txns count in pg_stat_replication_slots - [1] and adds the transaction size to the sum of bytes in the same repeatedly on the publisher, until the problem is solved. One of the good examples is duplication error on the subscriber side and this applies to both streaming and spill cases as well. This update prevents users from grasping the exact number and size of successful and unsuccessful transactions. Accordingly, we need to have new columns of failed transactions that will work to differentiate both of them for all types, which means spill, streaming and normal transactions. This will help users to measure the exact status of logical replication. Attached file is the POC patch for this. Current design is to save failed stats data in the ReplicationSlot struct. This is because after the error, I'm not able to access the ReorderBuffer object. Thus, I chose the object where I can interact with at the ReplicationSlotRelease timing. Below is one example that I can get on the publisher, after the duplication error on the subscriber caused by insert is solved. postgres=# select * from pg_stat_replication_slots; -[ RECORD 1 ]-------+------ slot_name | mysub spill_txns | 0 spill_count | 0 spill_bytes | 0 failed_spill_txns | 0 failed_spill_bytes | 0 stream_txns | 0 stream_count | 0 stream_bytes | 0 failed_stream_txns | 0 failed_stream_bytes | 0 total_txns | 4 total_bytes | 528 failed_total_txns | 3 failed_total_bytes | 396 stats_reset | Any ideas and comments are welcome. [1] - https://www.postgresql.org/docs/devel/monitoring-stats.html#MONITORING-PG-STAT-REPLICATION-SLOTS-VIE... Best Regards, Takamichi Osumi Attachments: [application/octet-stream] failed_transaction_stats_POC_v01.patch (17.1K, ../../OSBPR01MB48887CA8F40C8D984A6DC00CED199@OSBPR01MB4888.jpnprd01.prod.outlook.com/2-failed_transaction_stats_POC_v01.patch) download | inline diff: diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index 999d984..b35b667 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -962,11 +962,17 @@ CREATE VIEW pg_stat_replication_slots AS s.spill_txns, s.spill_count, s.spill_bytes, + s.failed_spill_txns, + s.failed_spill_bytes, s.stream_txns, s.stream_count, s.stream_bytes, + s.failed_stream_txns, + s.failed_stream_bytes, s.total_txns, s.total_bytes, + s.failed_total_txns, + s.failed_total_bytes, s.stats_reset FROM pg_replication_slots as r, LATERAL pg_stat_get_replication_slot(slot_name) as s diff --git a/src/backend/postmaster/pgstat.c b/src/backend/postmaster/pgstat.c index ce8888c..de98eb7 100644 --- a/src/backend/postmaster/pgstat.c +++ b/src/backend/postmaster/pgstat.c @@ -1819,11 +1819,17 @@ pgstat_report_replslot(const PgStat_StatReplSlotEntry *repSlotStat) msg.m_spill_txns = repSlotStat->spill_txns; msg.m_spill_count = repSlotStat->spill_count; msg.m_spill_bytes = repSlotStat->spill_bytes; + msg.m_failed_spill_txns = repSlotStat->failed_spill_txns; + msg.m_failed_spill_bytes = repSlotStat->failed_spill_bytes; msg.m_stream_txns = repSlotStat->stream_txns; msg.m_stream_count = repSlotStat->stream_count; msg.m_stream_bytes = repSlotStat->stream_bytes; + msg.m_failed_stream_txns = repSlotStat->failed_stream_txns; + msg.m_failed_stream_bytes = repSlotStat->failed_stream_bytes; msg.m_total_txns = repSlotStat->total_txns; msg.m_total_bytes = repSlotStat->total_bytes; + msg.m_failed_total_txns = repSlotStat->failed_total_txns; + msg.m_failed_total_bytes = repSlotStat->failed_total_bytes; pgstat_send(&msg, sizeof(PgStat_MsgReplSlot)); } @@ -5509,11 +5515,17 @@ pgstat_recv_replslot(PgStat_MsgReplSlot *msg, int len) slotent->spill_txns += msg->m_spill_txns; slotent->spill_count += msg->m_spill_count; slotent->spill_bytes += msg->m_spill_bytes; + slotent->failed_spill_txns += msg->m_failed_spill_txns; + slotent->failed_spill_bytes += msg->m_failed_spill_bytes; slotent->stream_txns += msg->m_stream_txns; slotent->stream_count += msg->m_stream_count; slotent->stream_bytes += msg->m_stream_bytes; + slotent->failed_stream_txns += msg->m_failed_stream_txns; + slotent->failed_stream_bytes += msg->m_failed_stream_bytes; slotent->total_txns += msg->m_total_txns; slotent->total_bytes += msg->m_total_bytes; + slotent->failed_total_txns += msg->m_failed_total_txns; + slotent->failed_total_bytes += msg->m_failed_total_bytes; } } } @@ -5760,11 +5772,17 @@ pgstat_reset_replslot(PgStat_StatReplSlotEntry *slotent, TimestampTz ts) slotent->spill_txns = 0; slotent->spill_count = 0; slotent->spill_bytes = 0; + slotent->failed_spill_txns = 0; + slotent->failed_spill_bytes = 0; slotent->stream_txns = 0; slotent->stream_count = 0; slotent->stream_bytes = 0; + slotent->failed_stream_txns = 0; + slotent->failed_stream_bytes = 0; slotent->total_txns = 0; slotent->total_bytes = 0; + slotent->failed_total_txns = 0; + slotent->failed_total_bytes = 0; slotent->stat_reset_timestamp = ts; } diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c index d536a5f..159223f 100644 --- a/src/backend/replication/logical/logical.c +++ b/src/backend/replication/logical/logical.c @@ -196,6 +196,12 @@ StartupDecodingContext(List *output_plugin_options, LWLockRelease(ProcArrayLock); } + slot->failedSpillTxns = 0; + slot->failedSpillBytes = 0; + slot->failedStreamTxns = 0; + slot->failedStreamBytes = 0; + slot->failedTotalTxns = 0; + slot->failedTotalBytes = 0; ctx->slot = slot; ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, xl_routine, ctx); @@ -1794,11 +1800,17 @@ UpdateDecodingStats(LogicalDecodingContext *ctx) repSlotStat.spill_txns = rb->spillTxns; repSlotStat.spill_count = rb->spillCount; repSlotStat.spill_bytes = rb->spillBytes; + repSlotStat.failed_spill_txns = 0; + repSlotStat.failed_spill_bytes = 0; repSlotStat.stream_txns = rb->streamTxns; repSlotStat.stream_count = rb->streamCount; repSlotStat.stream_bytes = rb->streamBytes; + repSlotStat.failed_stream_txns = 0; + repSlotStat.failed_stream_bytes = 0; repSlotStat.total_txns = rb->totalTxns; repSlotStat.total_bytes = rb->totalBytes; + repSlotStat.failed_total_txns = 0; + repSlotStat.failed_total_bytes = 0; pgstat_report_replslot(&repSlotStat); diff --git a/src/backend/replication/logical/reorderbuffer.c b/src/backend/replication/logical/reorderbuffer.c index b8c5e2a..ca35c42 100644 --- a/src/backend/replication/logical/reorderbuffer.c +++ b/src/backend/replication/logical/reorderbuffer.c @@ -2051,6 +2051,7 @@ ReorderBufferProcessTXN(ReorderBuffer *rb, ReorderBufferTXN *txn, PG_TRY(); { ReorderBufferChange *change; + ReplicationSlot *slot = MyReplicationSlot; if (using_subtxn) BeginInternalSubTransaction(streaming ? "stream" : "replay"); @@ -2409,9 +2410,13 @@ ReorderBufferProcessTXN(ReorderBuffer *rb, ReorderBufferTXN *txn, * which we have already accounted in ReorderBufferIterTXNNext. */ if (!rbtxn_is_streamed(txn)) + { rb->totalTxns++; + slot->failedTotalTxns++; + } rb->totalBytes += txn->total_size; + slot->failedTotalBytes += txn->total_size; /* * Done with current changes, send the last message for this set of @@ -3582,11 +3587,17 @@ ReorderBufferSerializeTXN(ReorderBuffer *rb, ReorderBufferTXN *txn) /* update the statistics iff we have spilled anything */ if (spilled) { + ReplicationSlot *slot = MyReplicationSlot; + int spill_txn; + rb->spillCount += 1; rb->spillBytes += size; + slot->failedSpillBytes += size; /* don't consider already serialized transactions */ - rb->spillTxns += (rbtxn_is_serialized(txn) || rbtxn_is_serialized_clear(txn)) ? 0 : 1; + spill_txn = (rbtxn_is_serialized(txn) || rbtxn_is_serialized_clear(txn)) ? 0 : 1; + rb->spillTxns += spill_txn; + slot->failedSpillTxns += spill_txn; /* update the decoding stats */ UpdateDecodingStats((LogicalDecodingContext *) rb->private_data); @@ -3861,6 +3872,7 @@ ReorderBufferStreamTXN(ReorderBuffer *rb, ReorderBufferTXN *txn) CommandId command_id; Size stream_bytes; bool txn_is_streamed; + ReplicationSlot *slot = MyReplicationSlot; /* We can never reach here for a subtransaction. */ Assert(txn->toptxn == NULL); @@ -3956,9 +3968,11 @@ ReorderBufferStreamTXN(ReorderBuffer *rb, ReorderBufferTXN *txn) rb->streamCount += 1; rb->streamBytes += stream_bytes; + slot->failedStreamBytes += stream_bytes; /* Don't consider already streamed transaction. */ rb->streamTxns += (txn_is_streamed) ? 0 : 1; + slot->failedStreamTxns += (txn_is_streamed) ? 0 : 1; /* update the decoding stats */ UpdateDecodingStats((LogicalDecodingContext *) rb->private_data); diff --git a/src/backend/replication/slot.c b/src/backend/replication/slot.c index 8c18b4e..ac5e953 100644 --- a/src/backend/replication/slot.c +++ b/src/backend/replication/slot.c @@ -44,6 +44,7 @@ #include "common/string.h" #include "miscadmin.h" #include "pgstat.h" +#include "replication/logical.h" #include "replication/slot.h" #include "storage/fd.h" #include "storage/proc.h" @@ -471,6 +472,11 @@ ReplicationSlotRelease(void) Assert(slot != NULL && slot->active_pid != 0); + if (SlotIsLogical(slot)) + { + UpdateDecodingFailedStats(); + } + if (slot->data.persistency == RS_EPHEMERAL) { /* @@ -1839,3 +1845,40 @@ RestoreSlotFromDisk(const char *name) (errmsg("too many replication slots active before shutdown"), errhint("Increase max_replication_slots and try again."))); } + +/* + * Report failed stats for a slot. + */ +void +UpdateDecodingFailedStats() +{ + PgStat_StatReplSlotEntry repSlotStat; + ReplicationSlot *slot = MyReplicationSlot; + + Assert(MyReplicationSlot != NULL); + + namestrcpy(&repSlotStat.slotname, NameStr(slot->data.name)); + repSlotStat.spill_txns = 0; + repSlotStat.spill_count = 0; + repSlotStat.spill_bytes = 0; + repSlotStat.failed_spill_txns = slot->failedSpillTxns; + repSlotStat.failed_spill_bytes = slot->failedSpillBytes; + repSlotStat.stream_txns = 0; + repSlotStat.stream_count = 0; + repSlotStat.stream_bytes = 0; + repSlotStat.failed_stream_txns = slot->failedStreamTxns; + repSlotStat.failed_stream_bytes = slot->failedStreamBytes; + repSlotStat.total_txns = 0; + repSlotStat.total_bytes = 0; + repSlotStat.failed_total_txns = slot->failedTotalTxns; + repSlotStat.failed_total_bytes = slot->failedTotalBytes; + + pgstat_report_replslot(&repSlotStat); + + slot->failedSpillTxns = 0; + slot->failedSpillBytes = 0; + slot->failedStreamTxns = 0; + slot->failedStreamBytes = 0; + slot->failedTotalTxns = 0; + slot->failedTotalBytes = 0; +} diff --git a/src/backend/utils/adt/pgstatfuncs.c b/src/backend/utils/adt/pgstatfuncs.c index f0e09ea..f518964 100644 --- a/src/backend/utils/adt/pgstatfuncs.c +++ b/src/backend/utils/adt/pgstatfuncs.c @@ -2313,7 +2313,7 @@ pg_stat_get_archiver(PG_FUNCTION_ARGS) Datum pg_stat_get_replication_slot(PG_FUNCTION_ARGS) { -#define PG_STAT_GET_REPLICATION_SLOT_COLS 10 +#define PG_STAT_GET_REPLICATION_SLOT_COLS 16 text *slotname_text = PG_GETARG_TEXT_P(0); NameData slotname; TupleDesc tupdesc; @@ -2336,17 +2336,29 @@ pg_stat_get_replication_slot(PG_FUNCTION_ARGS) INT8OID, -1, 0); TupleDescInitEntry(tupdesc, (AttrNumber) 4, "spill_bytes", INT8OID, -1, 0); - TupleDescInitEntry(tupdesc, (AttrNumber) 5, "stream_txns", + TupleDescInitEntry(tupdesc, (AttrNumber) 5, "failed_spill_txns", INT8OID, -1, 0); - TupleDescInitEntry(tupdesc, (AttrNumber) 6, "stream_count", + TupleDescInitEntry(tupdesc, (AttrNumber) 6, "failed_spill_bytes", INT8OID, -1, 0); - TupleDescInitEntry(tupdesc, (AttrNumber) 7, "stream_bytes", + TupleDescInitEntry(tupdesc, (AttrNumber) 7, "stream_txns", INT8OID, -1, 0); - TupleDescInitEntry(tupdesc, (AttrNumber) 8, "total_txns", + TupleDescInitEntry(tupdesc, (AttrNumber) 8, "stream_count", INT8OID, -1, 0); - TupleDescInitEntry(tupdesc, (AttrNumber) 9, "total_bytes", + TupleDescInitEntry(tupdesc, (AttrNumber) 9, "stream_bytes", INT8OID, -1, 0); - TupleDescInitEntry(tupdesc, (AttrNumber) 10, "stats_reset", + TupleDescInitEntry(tupdesc, (AttrNumber) 10, "failed_stream_txns", + INT8OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) 11, "failed_stream_bytes", + INT8OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) 12, "total_txns", + INT8OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) 13, "total_bytes", + INT8OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) 14, "failed_total_txns", + INT8OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) 15, "failed_total_bytes", + INT8OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) 16, "stats_reset", TIMESTAMPTZOID, -1, 0); BlessTupleDesc(tupdesc); @@ -2366,16 +2378,22 @@ pg_stat_get_replication_slot(PG_FUNCTION_ARGS) values[1] = Int64GetDatum(slotent->spill_txns); values[2] = Int64GetDatum(slotent->spill_count); values[3] = Int64GetDatum(slotent->spill_bytes); - values[4] = Int64GetDatum(slotent->stream_txns); - values[5] = Int64GetDatum(slotent->stream_count); - values[6] = Int64GetDatum(slotent->stream_bytes); - values[7] = Int64GetDatum(slotent->total_txns); - values[8] = Int64GetDatum(slotent->total_bytes); + values[4] = Int64GetDatum(slotent->failed_spill_txns); + values[5] = Int64GetDatum(slotent->failed_spill_bytes); + values[6] = Int64GetDatum(slotent->stream_txns); + values[7] = Int64GetDatum(slotent->stream_count); + values[8] = Int64GetDatum(slotent->stream_bytes); + values[9] = Int64GetDatum(slotent->failed_stream_txns); + values[10] = Int64GetDatum(slotent->failed_stream_bytes); + values[11] = Int64GetDatum(slotent->total_txns); + values[12] = Int64GetDatum(slotent->total_bytes); + values[13] = Int64GetDatum(slotent->failed_total_txns); + values[14] = Int64GetDatum(slotent->failed_total_bytes); if (slotent->stat_reset_timestamp == 0) - nulls[9] = true; + nulls[15] = true; else - values[9] = TimestampTzGetDatum(slotent->stat_reset_timestamp); + values[15] = TimestampTzGetDatum(slotent->stat_reset_timestamp); /* Returns the record as Datum */ PG_RETURN_DATUM(HeapTupleGetDatum(heap_form_tuple(tupdesc, values, nulls))); diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index fde251f..11cf01b 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -5313,9 +5313,9 @@ proname => 'pg_stat_get_replication_slot', prorows => '1', proisstrict => 'f', proretset => 't', provolatile => 's', proparallel => 'r', prorettype => 'record', proargtypes => 'text', - proallargtypes => '{text,text,int8,int8,int8,int8,int8,int8,int8,int8,timestamptz}', - proargmodes => '{i,o,o,o,o,o,o,o,o,o,o}', - proargnames => '{slot_name,slot_name,spill_txns,spill_count,spill_bytes,stream_txns,stream_count,stream_bytes,total_txns,total_bytes,stats_reset}', + proallargtypes => '{text,text,int8,int8,int8,int8,int8,int8,int8,int8,int8,int8,int8,int8,int8,int8,timestamptz}', + proargmodes => '{i,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o}', + proargnames => '{slot_name,slot_name,spill_txns,spill_count,spill_bytes,failed_spill_txns,failed_spill_bytes,stream_txns,stream_count,stream_bytes,failed_stream_txns,failed_stream_bytes,total_txns,total_bytes,failed_total_txns,failed_total_bytes,stats_reset}', prosrc => 'pg_stat_get_replication_slot' }, { oid => '6118', descr => 'statistics: information about subscription', proname => 'pg_stat_get_subscription', prorows => '10', proisstrict => 'f', diff --git a/src/include/pgstat.h b/src/include/pgstat.h index 9612c0a..5064408 100644 --- a/src/include/pgstat.h +++ b/src/include/pgstat.h @@ -532,11 +532,17 @@ typedef struct PgStat_MsgReplSlot PgStat_Counter m_spill_txns; PgStat_Counter m_spill_count; PgStat_Counter m_spill_bytes; + PgStat_Counter m_failed_spill_txns; + PgStat_Counter m_failed_spill_bytes; PgStat_Counter m_stream_txns; PgStat_Counter m_stream_count; PgStat_Counter m_stream_bytes; + PgStat_Counter m_failed_stream_txns; + PgStat_Counter m_failed_stream_bytes; PgStat_Counter m_total_txns; PgStat_Counter m_total_bytes; + PgStat_Counter m_failed_total_txns; + PgStat_Counter m_failed_total_bytes; } PgStat_MsgReplSlot; @@ -900,11 +906,17 @@ typedef struct PgStat_StatReplSlotEntry PgStat_Counter spill_txns; PgStat_Counter spill_count; PgStat_Counter spill_bytes; + PgStat_Counter failed_spill_txns; + PgStat_Counter failed_spill_bytes; PgStat_Counter stream_txns; PgStat_Counter stream_count; PgStat_Counter stream_bytes; + PgStat_Counter failed_stream_txns; + PgStat_Counter failed_stream_bytes; PgStat_Counter total_txns; PgStat_Counter total_bytes; + PgStat_Counter failed_total_txns; + PgStat_Counter failed_total_bytes; TimestampTz stat_reset_timestamp; } PgStat_StatReplSlotEntry; diff --git a/src/include/replication/slot.h b/src/include/replication/slot.h index 2eb7e3a..8d0bef6 100644 --- a/src/include/replication/slot.h +++ b/src/include/replication/slot.h @@ -164,6 +164,20 @@ typedef struct ReplicationSlot XLogRecPtr candidate_xmin_lsn; XLogRecPtr candidate_restart_valid; XLogRecPtr candidate_restart_lsn; + + /* + * Statistics about failed transactions. + * + * These failed statistics works to make a distinction between + * successful and unsuccessful transactions, by utilizing other + * corresponding stats of transactions respectively. + */ + int64 failedSpillTxns; + int64 failedSpillBytes; + int64 failedStreamTxns; + int64 failedStreamBytes; + int64 failedTotalTxns; + int64 failedTotalBytes; } ReplicationSlot; #define SlotIsPhysical(slot) ((slot)->data.database == InvalidOid) @@ -223,5 +237,6 @@ extern void StartupReplicationSlots(void); extern void CheckPointReplicationSlots(void); extern void CheckSlotRequirements(void); +extern void UpdateDecodingFailedStats(void); #endif /* SLOT_H */ diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out index e5ab112..41724bb 100644 --- a/src/test/regress/expected/rules.out +++ b/src/test/regress/expected/rules.out @@ -2054,14 +2054,20 @@ pg_stat_replication_slots| SELECT s.slot_name, s.spill_txns, s.spill_count, s.spill_bytes, + s.failed_spill_txns, + s.failed_spill_bytes, s.stream_txns, s.stream_count, s.stream_bytes, + s.failed_stream_txns, + s.failed_stream_bytes, s.total_txns, s.total_bytes, + s.failed_total_txns, + s.failed_total_bytes, s.stats_reset FROM pg_replication_slots r, - LATERAL pg_stat_get_replication_slot((r.slot_name)::text) s(slot_name, spill_txns, spill_count, spill_bytes, stream_txns, stream_count, stream_bytes, total_txns, total_bytes, stats_reset) + LATERAL pg_stat_get_replication_slot((r.slot_name)::text) s(slot_name, spill_txns, spill_count, spill_bytes, failed_spill_txns, failed_spill_bytes, stream_txns, stream_count, stream_bytes, failed_stream_txns, failed_stream_bytes, total_txns, total_bytes, failed_total_txns, failed_total_bytes, stats_reset) WHERE (r.datoid IS NOT NULL); pg_stat_slru| SELECT s.name, s.blks_zeroed, ^ permalink raw reply [nested|flat] 113+ messages in thread
* Re: Failed transaction statistics to measure the logical replication progress @ 2021-07-13 05:50 vignesh C <[email protected]> parent: [email protected] <[email protected]> 2 siblings, 1 reply; 113+ messages in thread From: vignesh C @ 2021-07-13 05:50 UTC (permalink / raw) To: [email protected] <[email protected]>; +Cc: [email protected] <[email protected]> On Thu, Jul 8, 2021 at 12:25 PM [email protected] <[email protected]> wrote: > > Hello, hackers > > > When the current HEAD fails during logical decoding, the failure > increments txns count in pg_stat_replication_slots - [1] and adds > the transaction size to the sum of bytes in the same repeatedly > on the publisher, until the problem is solved. > One of the good examples is duplication error on the subscriber side > and this applies to both streaming and spill cases as well. > > This update prevents users from grasping the exact number and size of > successful and unsuccessful transactions. Accordingly, we need to > have new columns of failed transactions that will work to differentiate > both of them for all types, which means spill, streaming and normal > transactions. This will help users to measure the exact status of > logical replication. > > Attached file is the POC patch for this. > Current design is to save failed stats data in the ReplicationSlot struct. > This is because after the error, I'm not able to access the ReorderBuffer object. > Thus, I chose the object where I can interact with at the ReplicationSlotRelease timing. > > Below is one example that I can get on the publisher, > after the duplication error on the subscriber caused by insert is solved. > > postgres=# select * from pg_stat_replication_slots; > -[ RECORD 1 ]-------+------ > slot_name | mysub > spill_txns | 0 > spill_count | 0 > spill_bytes | 0 > failed_spill_txns | 0 > failed_spill_bytes | 0 > stream_txns | 0 > stream_count | 0 > stream_bytes | 0 > failed_stream_txns | 0 > failed_stream_bytes | 0 > total_txns | 4 > total_bytes | 528 > failed_total_txns | 3 > failed_total_bytes | 396 > stats_reset | > > > Any ideas and comments are welcome. +1 for having logical replication failed statistics. Currently if there is any transaction failure in the subscriber after sending the decoded data to the subscriber like constraint violation, object not exist, the statistics will include the failed decoded transaction info and there is no way to identify the actual successful transaction data. This patch will help in measuring the actual decoded transaction data. Regards, Vignesh ^ permalink raw reply [nested|flat] 113+ messages in thread
* RE: Failed transaction statistics to measure the logical replication progress @ 2021-07-13 06:59 [email protected] <[email protected]> parent: vignesh C <[email protected]> 0 siblings, 0 replies; 113+ messages in thread From: [email protected] @ 2021-07-13 06:59 UTC (permalink / raw) To: 'vignesh C' <[email protected]>; +Cc: [email protected] <[email protected]> On Tuesday, July 13, 2021 2:50 PM vignesh C <[email protected]> wrote: > > When the current HEAD fails during logical decoding, the failure > > increments txns count in pg_stat_replication_slots - [1] and adds the > > transaction size to the sum of bytes in the same repeatedly on the > > publisher, until the problem is solved. > > One of the good examples is duplication error on the subscriber side > > and this applies to both streaming and spill cases as well. > > > > This update prevents users from grasping the exact number and size of > > successful and unsuccessful transactions. Accordingly, we need to have > > new columns of failed transactions that will work to differentiate > > both of them for all types, which means spill, streaming and normal > > transactions. This will help users to measure the exact status of > > logical replication. > > > > Attached file is the POC patch for this. > > Current design is to save failed stats data in the ReplicationSlot struct. > > This is because after the error, I'm not able to access the ReorderBuffer > object. > > Thus, I chose the object where I can interact with at the > ReplicationSlotRelease timing. > > Any ideas and comments are welcome. ... > +1 for having logical replication failed statistics. Currently if > there is any transaction failure in the subscriber after sending the decoded > data to the subscriber like constraint violation, object not exist, the statistics > will include the failed decoded transaction info and there is no way to identify > the actual successful transaction data. This patch will help in measuring the > actual decoded transaction data. Yeah, we can apply this improvement to other error cases. Thank you for sharing ideas to make this enhancement more persuasive. Best Regards, Takamichi Osumi ^ permalink raw reply [nested|flat] 113+ messages in thread
* Re: Failed transaction statistics to measure the logical replication progress @ 2021-07-27 06:59 Ajin Cherian <[email protected]> parent: [email protected] <[email protected]> 2 siblings, 1 reply; 113+ messages in thread From: Ajin Cherian @ 2021-07-27 06:59 UTC (permalink / raw) To: [email protected] <[email protected]>; +Cc: [email protected] <[email protected]> On Thu, Jul 8, 2021 at 4:55 PM [email protected] <[email protected]> wrote: > Attached file is the POC patch for this. > Current design is to save failed stats data in the ReplicationSlot struct. > This is because after the error, I'm not able to access the ReorderBuffer object. > Thus, I chose the object where I can interact with at the ReplicationSlotRelease timing. I think this is a good idea to capture the failed replication stats. But I'm wondering how you are deciding if the replication failed or not? Not all cases of ReplicationSLotRelease are due to a failure. It could also be due to a planned dropping of subscription or disable of subscription. I have not tested this but won't the failed stats be updated in this case as well? Is that correct? regards, Ajin Cherian Fujitsu Australia ^ permalink raw reply [nested|flat] 113+ messages in thread
* RE: Failed transaction statistics to measure the logical replication progress @ 2021-07-28 12:56 [email protected] <[email protected]> parent: Ajin Cherian <[email protected]> 0 siblings, 0 replies; 113+ messages in thread From: [email protected] @ 2021-07-28 12:56 UTC (permalink / raw) To: 'Ajin Cherian' <[email protected]>; +Cc: [email protected] <[email protected]> On Tuesday, July 27, 2021 3:59 PM Ajin Cherian <[email protected]> wrote: > On Thu, Jul 8, 2021 at 4:55 PM [email protected] > <[email protected]> wrote: > > > Attached file is the POC patch for this. > > Current design is to save failed stats data in the ReplicationSlot struct. > > This is because after the error, I'm not able to access the ReorderBuffer > object. > > Thus, I chose the object where I can interact with at the > ReplicationSlotRelease timing. > > I think this is a good idea to capture the failed replication stats. > But I'm wondering how you are deciding if the replication failed or not? Not all > cases of ReplicationSLotRelease are due to a failure. It could also be due to a > planned dropping of subscription or disable of subscription. I have not tested > this but won't the failed stats be updated in this case as well? Is that correct? Yes, what you said is true. Currently, when I run DROP SUBSCRIPTION or ALTER SUBSCRIPTION DISABLE, failed stats values are added to pg_stat_replication_slots unintentionally, if they have some left values. This is because all those commands, like the subscriber apply failure by duplication error, have the publisher get 'X' message at ProcessRepliesIfAny() and go into the path to call ReplicationSlotRelease(). Also, other opportunities like server stop call the same in the end, which leads to a situation that after the server restart, the value of failed stats catch up with the (successful) existing stats values. Accordingly, I need to change the patch to adjust those situations. Thank you. Best Regards, Takamichi Osumi ^ permalink raw reply [nested|flat] 113+ messages in thread
* Re: Failed transaction statistics to measure the logical replication progress @ 2021-07-29 01:49 Masahiko Sawada <[email protected]> parent: [email protected] <[email protected]> 2 siblings, 1 reply; 113+ messages in thread From: Masahiko Sawada @ 2021-07-29 01:49 UTC (permalink / raw) To: [email protected] <[email protected]>; +Cc: [email protected] <[email protected]> On Thu, Jul 8, 2021 at 3:55 PM [email protected] <[email protected]> wrote: > > Hello, hackers > > > When the current HEAD fails during logical decoding, the failure > increments txns count in pg_stat_replication_slots - [1] and adds > the transaction size to the sum of bytes in the same repeatedly > on the publisher, until the problem is solved. > One of the good examples is duplication error on the subscriber side > and this applies to both streaming and spill cases as well. > > This update prevents users from grasping the exact number and size of > successful and unsuccessful transactions. Accordingly, we need to > have new columns of failed transactions that will work to differentiate > both of them for all types, which means spill, streaming and normal > transactions. This will help users to measure the exact status of > logical replication. Could you please elaborate on use cases of the proposed statistics? For example, the current statistics on pg_replication_slots can be used for tuning logical_decoding_work_mem as well as inferring the total amount of bytes passed to the output plugin. How will the user use those statistics? Also, if we want the stats of successful transactions why don't we show the stats of successful transactions in the view instead of ones of failed transactions? > > Attached file is the POC patch for this. > Current design is to save failed stats data in the ReplicationSlot struct. > This is because after the error, I'm not able to access the ReorderBuffer object. > Thus, I chose the object where I can interact with at the ReplicationSlotRelease timing. When discussing the pg_stat_replication_slots view, there was an idea to store the slot statistics on ReplicationSlot struct. But the idea was rejected mainly because the struct is on the shared buffer[1]. If we store those counts on ReplicationSlot struct it increases the usage of shared memory. And those statistics are used only by logical slots and don’t necessarily need to be shared among the server processes. Moreover, if we want to add more statistics on the view in the future, it further increases the usage of shared memory. If we want to track the stats of successful transactions, I think it's easier to track them on the subscriber side rather than the publisher side. We can increase counters when applying [stream]commit/abort logical changes on the subscriber. Regards, [1] https://www.postgresql.org/message-id/CAA4eK1Kuj%2B3G59hh3wu86f4mmpQLpah_mGv2-wfAPyn%2BzT%3DP4A%40ma... -- Masahiko Sawada EDB: https://www.enterprisedb.com/ ^ permalink raw reply [nested|flat] 113+ messages in thread
* RE: Failed transaction statistics to measure the logical replication progress @ 2021-08-02 05:52 [email protected] <[email protected]> parent: Masahiko Sawada <[email protected]> 0 siblings, 1 reply; 113+ messages in thread From: [email protected] @ 2021-08-02 05:52 UTC (permalink / raw) To: 'Masahiko Sawada' <[email protected]>; +Cc: [email protected] <[email protected]> On Thursday, July 29, 2021 10:50 AM Masahiko Sawada <[email protected]> wrote: > On Thu, Jul 8, 2021 at 3:55 PM [email protected] > <[email protected]> wrote: > > When the current HEAD fails during logical decoding, the failure > > increments txns count in pg_stat_replication_slots - [1] and adds > > the transaction size to the sum of bytes in the same repeatedly on > > the publisher, until the problem is solved. > > One of the good examples is duplication error on the subscriber side > > and this applies to both streaming and spill cases as well. > > > > This update prevents users from grasping the exact number and size > > of successful and unsuccessful transactions. Accordingly, we need to > > have new columns of failed transactions that will work to > > differentiate both of them for all types, which means spill, > > streaming and normal transactions. This will help users to measure > > the exact status of logical replication. > > Could you please elaborate on use cases of the proposed statistics? > For example, the current statistics on pg_replication_slots can be > used for tuning logical_decoding_work_mem as well as inferring the > total amount of bytes passed to the output plugin. How will the user use those statistics? > > Also, if we want the stats of successful transactions why don't we > show the stats of successful transactions in the view instead of ones > of failed transactions? It works to show the ratio of successful and unsuccessful transactions, which should be helpful in terms of administrator perspective. FYI, the POC patch added the columns where I prefixed 'failed' to those names. But, substantially, it meant the ratio when user compared normal columns and newly introduced columns by this POC in the pg_stat_replication_slots. > > Attached file is the POC patch for this. > > Current design is to save failed stats data in the ReplicationSlot struct. > > This is because after the error, I'm not able to access the > > ReorderBuffer > object. > > Thus, I chose the object where I can interact with at the > ReplicationSlotRelease timing. > > When discussing the pg_stat_replication_slots view, there was an idea > to store the slot statistics on ReplicationSlot struct. But the idea > was rejected mainly because the struct is on the shared buffer[1]. If > we store those counts on ReplicationSlot struct it increases the usage > of shared memory. And those statistics are used only by logical slots > and don’t necessarily need to be shared among the server processes. Yes, I was aware of this. I was not sure if this design will be expected or not for the enhancement, I thought of changing the design accordingly once the idea gets accepted by the community. > Moreover, if we want to add more statistics on the view in the future, > it further increases the usage of shared memory. If we want to track > the stats of successful transactions, I think it's easier to track > them on the subscriber side rather than the publisher side. We can > increase counters when applying [stream]commit/abort logical changes on the subscriber. It's true that to track the stats of successful and unsuccessful transactions on the *sub* is easier than on the pub. After some survey, it turned out that I cannot distinguish the protocol messages between the cases of any failure (e.g. duplication error on the sub) from user intentional and successful operations(e.g. DROP SUBSCRIPTION and ALTER SUBSCRIPTION DISABLE) on the pub. If we truly want to achieve this change on the publisher side, protocol change requires in order to make above cases distinguishable, now I feel that it is better to do this in the subscriber side. Accordingly, I'm thinking to have unsuccessful and successful stats on the sub side. Sawada-san is now implementing a new view in [1]. Do you think that I should write a patch to introduce a new separate view or write a patch to add more columns to the new view "pg_stat_subscription_errors" that is added at [1] ? [1] - https://www.postgresql.org/message-id/CAD21AoDeScrsHhLyEPYqN3sydg6PxAPVBboK%3D30xJfUVihNZDA%40mail.g... Best Regards, Takamichi Osumi ^ permalink raw reply [nested|flat] 113+ messages in thread
* Re: Failed transaction statistics to measure the logical replication progress @ 2021-08-02 07:42 Masahiko Sawada <[email protected]> parent: [email protected] <[email protected]> 0 siblings, 2 replies; 113+ messages in thread From: Masahiko Sawada @ 2021-08-02 07:42 UTC (permalink / raw) To: [email protected] <[email protected]>; +Cc: [email protected] <[email protected]> On Mon, Aug 2, 2021 at 2:52 PM [email protected] <[email protected]> wrote: > > On Thursday, July 29, 2021 10:50 AM Masahiko Sawada <[email protected]> wrote: > > On Thu, Jul 8, 2021 at 3:55 PM [email protected] > > <[email protected]> wrote: > > > When the current HEAD fails during logical decoding, the failure > > > increments txns count in pg_stat_replication_slots - [1] and adds > > > the transaction size to the sum of bytes in the same repeatedly on > > > the publisher, until the problem is solved. > > > One of the good examples is duplication error on the subscriber side > > > and this applies to both streaming and spill cases as well. > > > > > > This update prevents users from grasping the exact number and size > > > of successful and unsuccessful transactions. Accordingly, we need to > > > have new columns of failed transactions that will work to > > > differentiate both of them for all types, which means spill, > > > streaming and normal transactions. This will help users to measure > > > the exact status of logical replication. > > > > Could you please elaborate on use cases of the proposed statistics? > > For example, the current statistics on pg_replication_slots can be > > used for tuning logical_decoding_work_mem as well as inferring the > > total amount of bytes passed to the output plugin. How will the user use those statistics? > > > > Also, if we want the stats of successful transactions why don't we > > show the stats of successful transactions in the view instead of ones > > of failed transactions? > It works to show the ratio of successful and unsuccessful transactions, > which should be helpful in terms of administrator perspective. > FYI, the POC patch added the columns where I prefixed 'failed' to those names. > But, substantially, it meant the ratio when user compared normal columns and > newly introduced columns by this POC in the pg_stat_replication_slots. What can the administrator use the ratio of successful and unsuccessful logical replication transactions for? For example, IIUC if a conflict happens on the subscriber as you mentioned, the successful transaction ratio calculated by those statistics is getting low, perhaps meaning the logical replication stopped. But it can be checked also by checking pg_stat_replication view or pg_replication_slots view (or error counts of pg_stat_subscription_errors view I’m proposing[1]). Do you have other use cases? > > Moreover, if we want to add more statistics on the view in the future, > > it further increases the usage of shared memory. If we want to track > > the stats of successful transactions, I think it's easier to track > > them on the subscriber side rather than the publisher side. We can > > increase counters when applying [stream]commit/abort logical changes on the subscriber. > It's true that to track the stats of successful and unsuccessful transactions on the *sub* > is easier than on the pub. After some survey, it turned out that I cannot distinguish > the protocol messages between the cases of any failure (e.g. duplication error on the sub) > from user intentional and successful operations(e.g. DROP SUBSCRIPTION and ALTER SUBSCRIPTION DISABLE) on the pub. > > If we truly want to achieve this change on the publisher side, > protocol change requires in order to make above cases distinguishable, > now I feel that it is better to do this in the subscriber side. > > Accordingly, I'm thinking to have unsuccessful and successful stats on the sub side. > Sawada-san is now implementing a new view in [1]. > Do you think that I should write a patch to introduce a new separate view > or write a patch to add more columns to the new view "pg_stat_subscription_errors" that is added at [1] ? pg_stat_subscriptions_errors view I'm proposing is a view showing the details of error happening during logical replication. So I think a separate view or pg_stat_subscription view would be a more appropriate place. Regards, [1] https://www.postgresql.org/message-id/flat/CAD21AoDeScrsHhLyEPYqN3sydg6PxAPVBboK=30xJfUVihNZDA@mail.... -- Masahiko Sawada EDB: https://www.enterprisedb.com/ ^ permalink raw reply [nested|flat] 113+ messages in thread
* Re: Failed transaction statistics to measure the logical replication progress @ 2021-08-03 02:47 Amit Kapila <[email protected]> parent: Masahiko Sawada <[email protected]> 1 sibling, 1 reply; 113+ messages in thread From: Amit Kapila @ 2021-08-03 02:47 UTC (permalink / raw) To: Masahiko Sawada <[email protected]>; +Cc: [email protected] <[email protected]>; [email protected] <[email protected]> On Mon, Aug 2, 2021 at 1:13 PM Masahiko Sawada <[email protected]> wrote: > > On Mon, Aug 2, 2021 at 2:52 PM [email protected] > <[email protected]> wrote: > > > > > > Accordingly, I'm thinking to have unsuccessful and successful stats on the sub side. > > Sawada-san is now implementing a new view in [1]. > > Do you think that I should write a patch to introduce a new separate view > > or write a patch to add more columns to the new view "pg_stat_subscription_errors" that is added at [1] ? > > pg_stat_subscriptions_errors view I'm proposing is a view showing the > details of error happening during logical replication. So I think a > separate view or pg_stat_subscription view would be a more appropriate > place. > +1 for having these stats in pg_stat_subscription. Do we want to add two columns (xact_commit: number of transactions successfully applied in this subscription, xact_rollback: number of transactions that have been rolled back in this subscription) or do you guys have something else in mind? -- With Regards, Amit Kapila. ^ permalink raw reply [nested|flat] 113+ messages in thread
* Re: Failed transaction statistics to measure the logical replication progress @ 2021-08-03 05:28 Masahiko Sawada <[email protected]> parent: Amit Kapila <[email protected]> 0 siblings, 2 replies; 113+ messages in thread From: Masahiko Sawada @ 2021-08-03 05:28 UTC (permalink / raw) To: Amit Kapila <[email protected]>; +Cc: [email protected] <[email protected]>; [email protected] <[email protected]> On Tue, Aug 3, 2021 at 11:47 AM Amit Kapila <[email protected]> wrote: > > On Mon, Aug 2, 2021 at 1:13 PM Masahiko Sawada <[email protected]> wrote: > > > > On Mon, Aug 2, 2021 at 2:52 PM [email protected] > > <[email protected]> wrote: > > > > > > > > > Accordingly, I'm thinking to have unsuccessful and successful stats on the sub side. > > > Sawada-san is now implementing a new view in [1]. > > > Do you think that I should write a patch to introduce a new separate view > > > or write a patch to add more columns to the new view "pg_stat_subscription_errors" that is added at [1] ? > > > > pg_stat_subscriptions_errors view I'm proposing is a view showing the > > details of error happening during logical replication. So I think a > > separate view or pg_stat_subscription view would be a more appropriate > > place. > > > > +1 for having these stats in pg_stat_subscription. Do we want to add > two columns (xact_commit: number of transactions successfully applied > in this subscription, xact_rollback: number of transactions that have > been rolled back in this subscription) Sounds good. We might want to have separate counters for the number of transactions failed due to an error and transactions rolled back by stream_abort. pg_stat_subscription currently shows logical replication worker stats on the shared memory. I think xact_commit and xact_rollback should survive beyond the server restarts, so it would be better to be collected by the stats collector. My skipping transaction patch adds a hash table of which entry represents a subscription stats. I guess we can use the hash table so that one subscription stats entry has both transaction stats and errors. > or do you guys have something else in mind? Osumi-san was originally concerned that there is no way to grasp the exact number and size of successful and unsuccessful transactions. The above idea covers only the number of successful and unsuccessful transactions but not the size. What do you think, Osumi-san? Regards, -- Masahiko Sawada EDB: https://www.enterprisedb.com/ ^ permalink raw reply [nested|flat] 113+ messages in thread
* Re: Failed transaction statistics to measure the logical replication progress @ 2021-08-03 05:29 vignesh C <[email protected]> parent: Masahiko Sawada <[email protected]> 1 sibling, 0 replies; 113+ messages in thread From: vignesh C @ 2021-08-03 05:29 UTC (permalink / raw) To: Masahiko Sawada <[email protected]>; +Cc: [email protected] <[email protected]>; [email protected] <[email protected]> On Mon, Aug 2, 2021 at 1:13 PM Masahiko Sawada <[email protected]> wrote: > > On Mon, Aug 2, 2021 at 2:52 PM [email protected] > <[email protected]> wrote: > > > > On Thursday, July 29, 2021 10:50 AM Masahiko Sawada <[email protected]> wrote: > > > On Thu, Jul 8, 2021 at 3:55 PM [email protected] > > > <[email protected]> wrote: > > > > When the current HEAD fails during logical decoding, the failure > > > > increments txns count in pg_stat_replication_slots - [1] and adds > > > > the transaction size to the sum of bytes in the same repeatedly on > > > > the publisher, until the problem is solved. > > > > One of the good examples is duplication error on the subscriber side > > > > and this applies to both streaming and spill cases as well. > > > > > > > > This update prevents users from grasping the exact number and size > > > > of successful and unsuccessful transactions. Accordingly, we need to > > > > have new columns of failed transactions that will work to > > > > differentiate both of them for all types, which means spill, > > > > streaming and normal transactions. This will help users to measure > > > > the exact status of logical replication. > > > > > > Could you please elaborate on use cases of the proposed statistics? > > > For example, the current statistics on pg_replication_slots can be > > > used for tuning logical_decoding_work_mem as well as inferring the > > > total amount of bytes passed to the output plugin. How will the user use those statistics? > > > > > > Also, if we want the stats of successful transactions why don't we > > > show the stats of successful transactions in the view instead of ones > > > of failed transactions? > > It works to show the ratio of successful and unsuccessful transactions, > > which should be helpful in terms of administrator perspective. > > FYI, the POC patch added the columns where I prefixed 'failed' to those names. > > But, substantially, it meant the ratio when user compared normal columns and > > newly introduced columns by this POC in the pg_stat_replication_slots. > > What can the administrator use the ratio of successful and > unsuccessful logical replication transactions for? For example, IIUC > if a conflict happens on the subscriber as you mentioned, the > successful transaction ratio calculated by those statistics is getting > low, perhaps meaning the logical replication stopped. But it can be > checked also by checking pg_stat_replication view or > pg_replication_slots view (or error counts of > pg_stat_subscription_errors view I’m proposing[1]). Do you have other > use cases? We could also include failed_data_size, this will help us to identify the actual bandwidth consumed by the failed transaction. It will help the DBA's to understand the network consumption in a better way. Currently only total transaction and total data will be available but when there is a failure, the failed transaction data will be sent repeatedly, if the DBA does not solve the actual cause of failure, there can be significant consumption of the network due to failure transaction being sent repeatedly. DBA will not be able to understand why there is so much network bandwidth consumption. If we give the failed transaction information, the DBA might not get alarmed in this case and understand that the network consumption is genuine. Also it will help monitoring tools to project this value. Regards, Vignesh ^ permalink raw reply [nested|flat] 113+ messages in thread
* RE: Failed transaction statistics to measure the logical replication progress @ 2021-08-03 06:09 [email protected] <[email protected]> parent: Masahiko Sawada <[email protected]> 1 sibling, 0 replies; 113+ messages in thread From: [email protected] @ 2021-08-03 06:09 UTC (permalink / raw) To: 'Masahiko Sawada' <[email protected]>; Amit Kapila <[email protected]>; +Cc: [email protected] <[email protected]> On Tuesday, August 3, 2021 2:29 PM Masahiko Sawada <[email protected]> wrote: > On Tue, Aug 3, 2021 at 11:47 AM Amit Kapila <[email protected]> > wrote: > > > > On Mon, Aug 2, 2021 at 1:13 PM Masahiko Sawada > <[email protected]> wrote: > > > > > > On Mon, Aug 2, 2021 at 2:52 PM [email protected] > > > <[email protected]> wrote: > > > > > > > > > > > > Accordingly, I'm thinking to have unsuccessful and successful stats on the > sub side. > > > > Sawada-san is now implementing a new view in [1]. > > > > Do you think that I should write a patch to introduce a new > > > > separate view or write a patch to add more columns to the new view > "pg_stat_subscription_errors" that is added at [1] ? > > > > > > pg_stat_subscriptions_errors view I'm proposing is a view showing > > > the details of error happening during logical replication. So I > > > think a separate view or pg_stat_subscription view would be a more > > > appropriate place. > > > > > > > +1 for having these stats in pg_stat_subscription. Do we want to add > > two columns (xact_commit: number of transactions successfully applied > > in this subscription, xact_rollback: number of transactions that have > > been rolled back in this subscription) > > Sounds good. We might want to have separate counters for the number of > transactions failed due to an error and transactions rolled back by stream_abort. Okay. I wanna make those separate as well for this feature. > pg_stat_subscription currently shows logical replication worker stats on the > shared memory. I think xact_commit and xact_rollback should survive beyond > the server restarts, so it would be better to be collected by the stats collector. > My skipping transaction patch adds a hash table of which entry represents a > subscription stats. I guess we can use the hash table so that one subscription > stats entry has both transaction stats and errors. > > > or do you guys have something else in mind? > > Osumi-san was originally concerned that there is no way to grasp the exact > number and size of successful and unsuccessful transactions. The above idea > covers only the number of successful and unsuccessful transactions but not the > size. What do you think, Osumi-san? Yeah, I think tracking sizes of failed transactions and roll-backed transactions is helpful to identify the genuine network consumption, as mentioned by Vignesh in another mail. I'd like to include those also and post a patch for this. Best Regards, Takamichi Osumi ^ permalink raw reply [nested|flat] 113+ messages in thread
* Re: Failed transaction statistics to measure the logical replication progress @ 2021-08-03 09:11 Amit Kapila <[email protected]> parent: Masahiko Sawada <[email protected]> 1 sibling, 1 reply; 113+ messages in thread From: Amit Kapila @ 2021-08-03 09:11 UTC (permalink / raw) To: Masahiko Sawada <[email protected]>; +Cc: [email protected] <[email protected]>; [email protected] <[email protected]> On Tue, Aug 3, 2021 at 10:59 AM Masahiko Sawada <[email protected]> wrote: > > On Tue, Aug 3, 2021 at 11:47 AM Amit Kapila <[email protected]> wrote: > > > > On Mon, Aug 2, 2021 at 1:13 PM Masahiko Sawada <[email protected]> wrote: > > > > > > On Mon, Aug 2, 2021 at 2:52 PM [email protected] > > > <[email protected]> wrote: > > > > > > > > > > > > Accordingly, I'm thinking to have unsuccessful and successful stats on the sub side. > > > > Sawada-san is now implementing a new view in [1]. > > > > Do you think that I should write a patch to introduce a new separate view > > > > or write a patch to add more columns to the new view "pg_stat_subscription_errors" that is added at [1] ? > > > > > > pg_stat_subscriptions_errors view I'm proposing is a view showing the > > > details of error happening during logical replication. So I think a > > > separate view or pg_stat_subscription view would be a more appropriate > > > place. > > > > > > > +1 for having these stats in pg_stat_subscription. Do we want to add > > two columns (xact_commit: number of transactions successfully applied > > in this subscription, xact_rollback: number of transactions that have > > been rolled back in this subscription) > > Sounds good. We might want to have separate counters for the number of > transactions failed due to an error and transactions rolled back by > stream_abort. > I was trying to think based on similar counters in pg_stat_database but if you think there is a value in showing errored and actual rollbacked transactions separately then we can do that but how do you think one can make use of it? -- With Regards, Amit Kapila. ^ permalink raw reply [nested|flat] 113+ messages in thread
* Re: Failed transaction statistics to measure the logical replication progress @ 2021-08-04 00:48 Masahiko Sawada <[email protected]> parent: Amit Kapila <[email protected]> 0 siblings, 1 reply; 113+ messages in thread From: Masahiko Sawada @ 2021-08-04 00:48 UTC (permalink / raw) To: Amit Kapila <[email protected]>; +Cc: [email protected] <[email protected]>; [email protected] <[email protected]> On Tue, Aug 3, 2021 at 6:11 PM Amit Kapila <[email protected]> wrote: > > On Tue, Aug 3, 2021 at 10:59 AM Masahiko Sawada <[email protected]> wrote: > > > > On Tue, Aug 3, 2021 at 11:47 AM Amit Kapila <[email protected]> wrote: > > > > > > On Mon, Aug 2, 2021 at 1:13 PM Masahiko Sawada <[email protected]> wrote: > > > > > > > > On Mon, Aug 2, 2021 at 2:52 PM [email protected] > > > > <[email protected]> wrote: > > > > > > > > > > > > > > > Accordingly, I'm thinking to have unsuccessful and successful stats on the sub side. > > > > > Sawada-san is now implementing a new view in [1]. > > > > > Do you think that I should write a patch to introduce a new separate view > > > > > or write a patch to add more columns to the new view "pg_stat_subscription_errors" that is added at [1] ? > > > > > > > > pg_stat_subscriptions_errors view I'm proposing is a view showing the > > > > details of error happening during logical replication. So I think a > > > > separate view or pg_stat_subscription view would be a more appropriate > > > > place. > > > > > > > > > > +1 for having these stats in pg_stat_subscription. Do we want to add > > > two columns (xact_commit: number of transactions successfully applied > > > in this subscription, xact_rollback: number of transactions that have > > > been rolled back in this subscription) > > > > Sounds good. We might want to have separate counters for the number of > > transactions failed due to an error and transactions rolled back by > > stream_abort. > > > > I was trying to think based on similar counters in pg_stat_database > but if you think there is a value in showing errored and actual > rollbacked transactions separately then we can do that but how do you > think one can make use of it? I'm concerned that the value that includes both errored and actual rollbacked transactions doesn't make sense in practice since the number of errored transactions can easily get increased once a conflict happens. IMO the errored transaction doesn’t not necessarily necessary since the number of (successive) errors that happened on the subscription is tracked by pg_stat_subscription_errors view. So it might be enough to have actual rollbacked transactions. If this value is high, it's likely that many rollbacked transactions are streamed, unnecessarily consuming network bandwidth. So the user might want to increase logical_decoding_work_mem to suppress transactions getting streamed. Regards, -- Masahiko Sawada EDB: https://www.enterprisedb.com/ ^ permalink raw reply [nested|flat] 113+ messages in thread
* Re: Failed transaction statistics to measure the logical replication progress @ 2021-08-04 03:17 Amit Kapila <[email protected]> parent: Masahiko Sawada <[email protected]> 0 siblings, 1 reply; 113+ messages in thread From: Amit Kapila @ 2021-08-04 03:17 UTC (permalink / raw) To: Masahiko Sawada <[email protected]>; +Cc: [email protected] <[email protected]>; [email protected] <[email protected]> On Wed, Aug 4, 2021 at 6:19 AM Masahiko Sawada <[email protected]> wrote: > > On Tue, Aug 3, 2021 at 6:11 PM Amit Kapila <[email protected]> wrote: > > > > I was trying to think based on similar counters in pg_stat_database > > but if you think there is a value in showing errored and actual > > rollbacked transactions separately then we can do that but how do you > > think one can make use of it? > > I'm concerned that the value that includes both errored and actual > rollbacked transactions doesn't make sense in practice since the > number of errored transactions can easily get increased once a > conflict happens. IMO the errored transaction doesn’t not necessarily > necessary since the number of (successive) errors that happened on the > subscription is tracked by pg_stat_subscription_errors view. > It sounds awkward to display two of the xact (xact_commit, xact_rollback) counters in one view and then the other similar counter (xact_error or something like that) in another view. Isn't it better to display all of them together possibly in pg_stat_subscription? I guess it might be a bit tricky to track counters for tablesync workers separately but we can track them in the corresponding subscription. > So it > might be enough to have actual rollbacked transactions. If this value > is high, it's likely that many rollbacked transactions are streamed, > unnecessarily consuming network bandwidth. So the user might want to > increase logical_decoding_work_mem to suppress transactions getting > streamed. > Okay, we might want to probably document such a use case. -- With Regards, Amit Kapila. ^ permalink raw reply [nested|flat] 113+ messages in thread
* Re: Failed transaction statistics to measure the logical replication progress @ 2021-08-04 07:04 Masahiko Sawada <[email protected]> parent: Amit Kapila <[email protected]> 0 siblings, 1 reply; 113+ messages in thread From: Masahiko Sawada @ 2021-08-04 07:04 UTC (permalink / raw) To: Amit Kapila <[email protected]>; +Cc: [email protected] <[email protected]>; [email protected] <[email protected]> On Wed, Aug 4, 2021 at 12:17 PM Amit Kapila <[email protected]> wrote: > > On Wed, Aug 4, 2021 at 6:19 AM Masahiko Sawada <[email protected]> wrote: > > > > On Tue, Aug 3, 2021 at 6:11 PM Amit Kapila <[email protected]> wrote: > > > > > > I was trying to think based on similar counters in pg_stat_database > > > but if you think there is a value in showing errored and actual > > > rollbacked transactions separately then we can do that but how do you > > > think one can make use of it? > > > > I'm concerned that the value that includes both errored and actual > > rollbacked transactions doesn't make sense in practice since the > > number of errored transactions can easily get increased once a > > conflict happens. IMO the errored transaction doesn’t not necessarily > > necessary since the number of (successive) errors that happened on the > > subscription is tracked by pg_stat_subscription_errors view. > > > > It sounds awkward to display two of the xact (xact_commit, > xact_rollback) counters in one view and then the other similar counter > (xact_error or something like that) in another view. Isn't it better > to display all of them together possibly in pg_stat_subscription? I > guess it might be a bit tricky to track counters for tablesync workers > separately but we can track them in the corresponding subscription. I meant that the number of rolled back transactions due to an error seems not to be necessary since pg_stat_subscription_errors has a similar value. So what I imagined is that we have xact_commit and xact_rollback (counting only actual rollbacked transaction) counters in pg_stat_subscription. What do you think of this idea? Or did you mean the number of errored transactions also has value so should be included in pg_stat_subscription along with xact_commit and xact_rollback? Originally I thought your proposal of having the number of rollback transactions includes both errored transactions and actual rolled back transactions so my point was it's better to separate them and include only the actual rolled-back transaction. Regards, -- Masahiko Sawada EDB: https://www.enterprisedb.com/ ^ permalink raw reply [nested|flat] 113+ messages in thread
* Re: Failed transaction statistics to measure the logical replication progress @ 2021-08-04 12:58 Amit Kapila <[email protected]> parent: Masahiko Sawada <[email protected]> 0 siblings, 1 reply; 113+ messages in thread From: Amit Kapila @ 2021-08-04 12:58 UTC (permalink / raw) To: Masahiko Sawada <[email protected]>; +Cc: [email protected] <[email protected]>; [email protected] <[email protected]> On Wed, Aug 4, 2021 at 12:35 PM Masahiko Sawada <[email protected]> wrote: > > On Wed, Aug 4, 2021 at 12:17 PM Amit Kapila <[email protected]> wrote: > > > > On Wed, Aug 4, 2021 at 6:19 AM Masahiko Sawada <[email protected]> wrote: > > > > > > On Tue, Aug 3, 2021 at 6:11 PM Amit Kapila <[email protected]> wrote: > > > > > > > > I was trying to think based on similar counters in pg_stat_database > > > > but if you think there is a value in showing errored and actual > > > > rollbacked transactions separately then we can do that but how do you > > > > think one can make use of it? > > > > > > I'm concerned that the value that includes both errored and actual > > > rollbacked transactions doesn't make sense in practice since the > > > number of errored transactions can easily get increased once a > > > conflict happens. IMO the errored transaction doesn’t not necessarily > > > necessary since the number of (successive) errors that happened on the > > > subscription is tracked by pg_stat_subscription_errors view. > > > > > > > It sounds awkward to display two of the xact (xact_commit, > > xact_rollback) counters in one view and then the other similar counter > > (xact_error or something like that) in another view. Isn't it better > > to display all of them together possibly in pg_stat_subscription? I > > guess it might be a bit tricky to track counters for tablesync workers > > separately but we can track them in the corresponding subscription. > > I meant that the number of rolled back transactions due to an error > seems not to be necessary since pg_stat_subscription_errors has a > similar value. > I got that point. > So what I imagined is that we have xact_commit and > xact_rollback (counting only actual rollbacked transaction) counters > in pg_stat_subscription. What do you think of this idea? Or did you > mean the number of errored transactions also has value so should be > included in pg_stat_subscription along with xact_commit and > xact_rollback? > I meant the later one. I think it might be better to display all three (xact_commit, xact_abort, xact_error) in one place. Earlier it made sense to me to display it in pg_stat_subscription_errors but not sure after this proposal. Won't it be better for users to see all the counters in one view? > Originally I thought your proposal of having the number of rollback > transactions includes both errored transactions and actual rolled back > transactions so my point was it's better to separate them and include > only the actual rolled-back transaction. > I am fine with your proposal to separate the actual rollback and error transactions counter but I thought it would be better to display them in one view. -- With Regards, Amit Kapila. ^ permalink raw reply [nested|flat] 113+ messages in thread
* RE: Failed transaction statistics to measure the logical replication progress @ 2021-09-02 02:22 [email protected] <[email protected]> parent: Amit Kapila <[email protected]> 0 siblings, 1 reply; 113+ messages in thread From: [email protected] @ 2021-09-02 02:22 UTC (permalink / raw) To: 'Amit Kapila' <[email protected]>; Masahiko Sawada <[email protected]>; +Cc: [email protected] <[email protected]> Hi I've made a new patch to extend pg_stat_subscription as suggested in [1] to have columns xact_commit, xact_error and independent xact_abort mentioned in [2]. Also, during discussion, we touched a topic if we should include data sizes for each column above and concluded that it's better to have ones. Accordingly, I've implemented corresponding columns to show the data sizes as well. Note that this patch depends on v12 patchset of apply error callback provided in [3]. Therfore, applying the patchset first is required, if you would like to test my patch. [1] - https://www.postgresql.org/message-id/CAA4eK1%2BtOV-%2BssGjj1zq%2BnAL8a9LfPsxbtyupZGvZ0U7nV0A7g%40ma... [2] - https://www.postgresql.org/message-id/CAA4eK1KMT8biciVqTBoZ9gYV-Gf297JFeNhJaxZNmFrZL8m2jA%40mail.gma... [3] - https://www.postgresql.org/message-id/CAD21AoA5HrhXqwbYLpSobGzV6rWoJmH3-NB9J3YarKDwARBj4w%40mail.gma... Best Regards, Takamichi Osumi Attachments: [application/octet-stream] extend_subscription_stats_of_transaction_v02.patch (24.0K, ../../TY2PR01MB48905729E8CB81805CCF64C0EDCE9@TY2PR01MB4890.jpnprd01.prod.outlook.com/2-extend_subscription_stats_of_transaction_v02.patch) download | inline diff: diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml index 0c02e46..ee36c3b 100644 --- a/doc/src/sgml/monitoring.sgml +++ b/doc/src/sgml/monitoring.sgml @@ -3046,6 +3046,60 @@ SELECT pid, wait_event_type, wait_event FROM pg_stat_activity WHERE wait_event i <row> <entry role="catalog_table_entry"><para role="column_definition"> + <structfield>xact_commit</structfield> <type>bigint</type> + </para> + <para> + Number of transactions successfully applied in this subscription + </para></entry> + </row> + + <row> + <entry role="catalog_table_entry"><para role="column_definition"> + <structfield>xact_commit_bytes</structfield> <type>bigint</type> + </para> + <para> + Amount of transactions data successfully applied in this subscription + </para></entry> + </row> + + <row> + <entry role="catalog_table_entry"><para role="column_definition"> + <structfield>xact_error</structfield> <type>bigint</type> + </para> + <para> + Number of transactions unsuccessfully applied in this subscription + </para></entry> + </row> + + <row> + <entry role="catalog_table_entry"><para role="column_definition"> + <structfield>xact_error_bytes</structfield> <type>bigint</type> + </para> + <para> + Amount of transactions data unsuccessfully applied in this subscription + </para></entry> + </row> + + <row> + <entry role="catalog_table_entry"><para role="column_definition"> + <structfield>xact_abort</structfield> <type>bigint</type> + </para> + <para> + Number of transactions aborted in this subscription + </para></entry> + </row> + + <row> + <entry role="catalog_table_entry"><para role="column_definition"> + <structfield>xact_abort_bytes</structfield> <type>bigint</type> + </para> + <para> + Amount of transactions data aborted in this subscription + </para></entry> + </row> + + <row> + <entry role="catalog_table_entry"><para role="column_definition"> <structfield>latest_end_time</structfield> <type>timestamp with time zone</type> </para> <para> diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index 449692a..109ba92 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -909,10 +909,37 @@ CREATE VIEW pg_stat_subscription AS st.last_msg_send_time, st.last_msg_receipt_time, st.latest_end_lsn, + pg_stat_get_subscription_xact_commit(su.oid, st.relid) AS xact_commit, + pg_stat_get_subscription_xact_commit_bytes(su.oid, st.relid) AS xact_commit_bytes, + pg_stat_get_subscription_xact_error(su.oid, st.relid) AS xact_error, + pg_stat_get_subscription_xact_error_bytes(su.oid, st.relid) AS xact_error_bytes, + pg_stat_get_subscription_xact_abort(su.oid, st.relid) AS xact_abort, + pg_stat_get_subscription_xact_abort_bytes(su.oid, st.relid) AS xact_abort_bytes, st.latest_end_time FROM pg_subscription su - LEFT JOIN pg_stat_get_subscription(NULL) st - ON (st.subid = su.oid); + LEFT JOIN + (SELECT + s.subid, + s.relid, + s.pid, + s.received_lsn, + s.last_msg_send_time, + s.last_msg_receipt_time, + s.latest_end_lsn, + s.latest_end_time + FROM pg_stat_get_subscription(NULL) s + UNION -- acquire relid of table sync worker + SELECT + r.srsubid, + r.srrelid, + NULL as pid, + NULL as received_lsn, + NULL as last_msg_send_time, + NULL as last_msg_receipt_time, + NULL as latest_end_lsn, + NULL as latest_end_time + FROM pg_subscription_rel r WHERE r.srsubstate <> 'r') st + ON (st.subid = su.oid); CREATE VIEW pg_stat_ssl AS SELECT diff --git a/src/backend/postmaster/pgstat.c b/src/backend/postmaster/pgstat.c index 778e409..bca3c41 100644 --- a/src/backend/postmaster/pgstat.c +++ b/src/backend/postmaster/pgstat.c @@ -55,6 +55,7 @@ #include "postmaster/postmaster.h" #include "replication/slot.h" #include "replication/walsender.h" +#include "replication/logicalworker.h" #include "storage/backendid.h" #include "storage/dsm.h" #include "storage/fd.h" @@ -378,6 +379,7 @@ static void pgstat_recv_checksum_failure(PgStat_MsgChecksumFailure *msg, int len static void pgstat_recv_connstat(PgStat_MsgConn *msg, int len); static void pgstat_recv_replslot(PgStat_MsgReplSlot *msg, int len); static void pgstat_recv_tempfile(PgStat_MsgTempFile *msg, int len); +static void pgstat_recv_subscription_success(PgStat_MsgSubscriptionErr *msg, int len); static void pgstat_recv_subscription_error(PgStat_MsgSubscriptionErr *msg, int len); static void pgstat_recv_subscription_error_purge(PgStat_MsgSubscriptionErrPurge *msg, int len); @@ -2038,6 +2040,40 @@ pgstat_report_replslot_drop(const char *slotname) } /* ---------- + * pgstat_report_subscription_success() - + * + * Tell the collector about the subscription success. + * ---------- + */ +void +pgstat_report_subscription_success(Oid subid, Oid subrel, + LogicalRepMsgType command, PgStat_Counter bytes) +{ + PgStat_MsgSubscriptionErr msg; + + Assert(command == 0 /* table sync worker */ || + command == LOGICAL_REP_MSG_COMMIT || + command == LOGICAL_REP_MSG_COMMIT_PREPARED || + command == LOGICAL_REP_MSG_STREAM_ABORT); + + pgstat_setheader(&msg.m_hdr, PGSTAT_MTYPE_SUBSCRIPTIONSUCCESS); + msg.m_subid = subid; + msg.m_subrelid = subrel; + msg.m_clear = false; + msg.m_reset = false; + msg.m_databaseid = InvalidOid; + msg.m_relid = InvalidOid; + msg.m_command = command; + msg.m_xid = InvalidTransactionId; + msg.m_bytes = bytes; + msg.m_failure_time = GetCurrentTimestamp(); + msg.m_errmsg[0] = '\0'; + pgstat_send(&msg, sizeof(PgStat_MsgSubscriptionErr)); + + reset_apply_error_context_xact_size(); +} + +/* ---------- * pgstat_report_subscription_error() - * * Tell the collector about the subscription error. @@ -2063,10 +2099,13 @@ pgstat_report_subscription_error(Oid subid, Oid subrelid, Oid relid, msg.m_relid = relid; msg.m_command = command; msg.m_xid = xid; + msg.m_bytes = get_apply_error_context_xact_size(); msg.m_failure_time = GetCurrentTimestamp(); strlcpy(msg.m_errmsg, errmsg, PGSTAT_SUBSCRIPTIONERR_MSGLEN); pgstat_send(&msg, len); + + reset_apply_error_context_xact_size(); } /* ---------- @@ -3730,6 +3769,10 @@ PgstatCollectorMain(int argc, char *argv[]) pgstat_recv_connstat(&msg.msg_conn, len); break; + case PGSTAT_MTYPE_SUBSCRIPTIONSUCCESS: + pgstat_recv_subscription_success(&msg.msg_subscriptionerr, len); + break; + case PGSTAT_MTYPE_SUBSCRIPTIONERR: pgstat_recv_subscription_error(&msg.msg_subscriptionerr, len); break; @@ -6153,6 +6196,45 @@ pgstat_recv_funcpurge(PgStat_MsgFuncpurge *msg, int len) } /* ---------- + * pgstat_recv_subscription_success() - + * + * Process a SUBSCRIPTIONERR message. + * ---------- + */ +static void +pgstat_recv_subscription_success(PgStat_MsgSubscriptionErr *msg, int len) +{ + PgStat_StatSubErrEntry *errent; + bool create = !(msg->m_reset || msg->m_clear); + + errent = pgstat_get_subscription_error_entry(msg->m_subid, + msg->m_subrelid, + create); + + /* msg from table sync worker */ + if (msg->m_command == 0) + errent->xact_commit++; + else + { + switch(msg->m_command) + { + case LOGICAL_REP_MSG_COMMIT: + case LOGICAL_REP_MSG_COMMIT_PREPARED: + errent->xact_commit++; + errent->xact_commit_bytes += msg->m_bytes; + break; + case LOGICAL_REP_MSG_STREAM_ABORT: + errent->xact_abort++; + errent->xact_abort_bytes = msg->m_bytes; + break; + default: + elog(ERROR, "unexpected command type"); + } + } + errent->last_failure = msg->m_failure_time; +} + +/* ---------- * pgstat_recv_subscription_error() - * * Process a SUBSCRIPTIONERR message. @@ -6202,6 +6284,8 @@ pgstat_recv_subscription_error(PgStat_MsgSubscriptionErr *msg, int len) errent->relid = msg->m_relid; errent->command = msg->m_command; errent->xid = msg->m_xid; + errent->xact_error++; + errent->xact_error_bytes += msg->m_bytes; errent->failure_count++; errent->last_failure = msg->m_failure_time; strlcpy(errent->last_errmsg, msg->m_errmsg, PGSTAT_SUBSCRIPTIONERR_MSGLEN); @@ -6465,6 +6549,12 @@ pgstat_get_subscription_error_entry(Oid subid, Oid subrelid, bool create) errent->relid = InvalidOid; errent->command = 0; errent->xid = InvalidTransactionId; + errent->xact_commit = 0; + errent->xact_commit_bytes = 0; + errent->xact_error = 0; + errent->xact_error_bytes = 0; + errent->xact_abort = 0; + errent->xact_abort_bytes = 0; errent->failure_count = 0; errent->last_failure = 0; errent->last_errmsg[0] = '\0'; diff --git a/src/backend/replication/logical/proto.c b/src/backend/replication/logical/proto.c index 9f5bf4b..64d18c6 100644 --- a/src/backend/replication/logical/proto.c +++ b/src/backend/replication/logical/proto.c @@ -17,6 +17,7 @@ #include "catalog/pg_type.h" #include "libpq/pqformat.h" #include "replication/logicalproto.h" +#include "replication/logicalworker.h" #include "utils/lsyscache.h" #include "utils/syscache.h" @@ -872,6 +873,9 @@ logicalrep_read_tuple(StringInfo in, LogicalRepTupleData *tuple) case LOGICALREP_COLUMN_TEXT: len = pq_getmsgint(in, 4); /* read length */ + /* accumulate data length for transaction stats */ + add_apply_error_context_xact_size(len); + /* and data */ value->data = palloc(len + 1); pq_copymsgbytes(in, value->data, len); @@ -884,6 +888,9 @@ logicalrep_read_tuple(StringInfo in, LogicalRepTupleData *tuple) case LOGICALREP_COLUMN_BINARY: len = pq_getmsgint(in, 4); /* read length */ + /* accumulate data length for transaction stats */ + add_apply_error_context_xact_size(len); + /* and data */ value->data = palloc(len + 1); pq_copymsgbytes(in, value->data, len); diff --git a/src/backend/replication/logical/tablesync.c b/src/backend/replication/logical/tablesync.c index f07983a..7ba7486 100644 --- a/src/backend/replication/logical/tablesync.c +++ b/src/backend/replication/logical/tablesync.c @@ -1150,6 +1150,15 @@ copy_table_done: SpinLockRelease(&MyLogicalRepWorker->relmutex); /* + * Update the stats for table sync. We don't record the bytes of + * table synchronization. + */ + pgstat_report_subscription_success(MyLogicalRepWorker->subid, + MyLogicalRepWorker->relid, + 0, /* no corresponding logical message type */ + 0); + + /* * Finally, wait until the main apply worker tells us to catch up and then * return to let LogicalRepApplyLoop do it. */ diff --git a/src/backend/replication/logical/worker.c b/src/backend/replication/logical/worker.c index 34ed8e4..a3f9ce3 100644 --- a/src/backend/replication/logical/worker.c +++ b/src/backend/replication/logical/worker.c @@ -222,6 +222,9 @@ typedef struct ApplyErrorCallbackArg LogicalRepMsgType command; /* 0 if invalid */ LogicalRepRelMapEntry *rel; + /* data size of this transaction */ + PgStat_Counter bytes; + /* Remote node information */ int remote_attnum; /* -1 if invalid */ TransactionId remote_xid; @@ -232,6 +235,7 @@ static ApplyErrorCallbackArg apply_error_callback_arg = { .command = 0, .rel = NULL, + .bytes = 0, .remote_attnum = -1, .remote_xid = InvalidTransactionId, .ts = 0, @@ -863,6 +867,10 @@ apply_handle_commit(StringInfo s) process_syncing_tables(commit_data.end_lsn); pgstat_report_activity(STATE_IDLE, NULL); + pgstat_report_subscription_success(MySubscription->oid, + InvalidOid, + LOGICAL_REP_MSG_COMMIT, + apply_error_callback_arg.bytes); reset_apply_error_context_info(); } @@ -1016,6 +1024,10 @@ apply_handle_commit_prepared(StringInfo s) process_syncing_tables(prepare_data.end_lsn); pgstat_report_activity(STATE_IDLE, NULL); + pgstat_report_subscription_success(MySubscription->oid, + InvalidOid, + LOGICAL_REP_MSG_COMMIT_PREPARED, + apply_error_callback_arg.bytes); reset_apply_error_context_info(); } @@ -1350,6 +1362,10 @@ apply_handle_stream_abort(StringInfo s) if (is_skipping_changes()) stop_skipping_changes(InvalidXLogRecPtr, 0); + pgstat_report_subscription_success(MySubscription->oid, + InvalidOid, + LOGICAL_REP_MSG_STREAM_ABORT, + apply_error_callback_arg.bytes); reset_apply_error_context_info(); } @@ -3380,6 +3396,9 @@ stream_write_change(char action, StringInfo s) len = (s->len - s->cursor); BufFileWrite(stream_fd, &s->data[s->cursor], len); + + /* accumulate the bytes for stats */ + add_apply_error_context_xact_size(len); } /* @@ -3744,6 +3763,27 @@ set_apply_error_context_xact(TransactionId xid, TimestampTz ts) apply_error_callback_arg.ts = ts; } +/* Exported so that stats collector can utilize this value */ +int64 +get_apply_error_context_xact_size(void) +{ + return apply_error_callback_arg.bytes; +} + +/* Add size to apply error callback bytes */ +void +add_apply_error_context_xact_size(int64 size) +{ + apply_error_callback_arg.bytes += size; +} + +/* Reset bytes information of apply error callback */ +void +reset_apply_error_context_xact_size(void) +{ + apply_error_callback_arg.bytes = 0; +} + /* Reset all information of apply error callback */ static inline void reset_apply_error_context_info(void) diff --git a/src/backend/utils/adt/pgstatfuncs.c b/src/backend/utils/adt/pgstatfuncs.c index c454e2f..0b4ad21 100644 --- a/src/backend/utils/adt/pgstatfuncs.c +++ b/src/backend/utils/adt/pgstatfuncs.c @@ -2492,3 +2492,136 @@ pg_stat_get_subscription_error(PG_FUNCTION_ARGS) /* Returns the record as Datum */ PG_RETURN_DATUM(HeapTupleGetDatum(heap_form_tuple(tupdesc, values, nulls))); } + +Datum +pg_stat_get_subscription_xact_commit(PG_FUNCTION_ARGS) +{ + Oid subid = PG_GETARG_OID(0); + Oid relid; + int64 result; + PgStat_StatSubErrEntry *errent; + + if (PG_ARGISNULL(1)) + relid = InvalidOid; + else + relid = PG_GETARG_OID(1); + + /* Get subscription error entry */ + if ((errent = pgstat_fetch_subscription_error(subid, relid)) == NULL) + result = 0; + else + result = (int64) (errent->xact_commit); + + PG_RETURN_INT64(result); +} + +Datum +pg_stat_get_subscription_xact_commit_bytes(PG_FUNCTION_ARGS) +{ + Oid subid = PG_GETARG_OID(0); + Oid relid; + int64 result; + PgStat_StatSubErrEntry *errent; + + if (PG_ARGISNULL(1)) + relid = InvalidOid; + else + relid = PG_GETARG_OID(1); + + /* Get subscription error entry */ + if ((errent = pgstat_fetch_subscription_error(subid, relid)) == NULL) + result = 0; + else + result = (int64) (errent->xact_commit_bytes); + + PG_RETURN_INT64(result); +} + + +Datum +pg_stat_get_subscription_xact_error(PG_FUNCTION_ARGS) +{ + Oid subid = PG_GETARG_OID(0); + Oid relid; + int64 result; + PgStat_StatSubErrEntry *errent; + + if (PG_ARGISNULL(1)) + relid = InvalidOid; + else + relid = PG_GETARG_OID(1); + + /* Get subscription error entry */ + if ((errent = pgstat_fetch_subscription_error(subid, relid)) == NULL) + result = 0; + else + result = (int64) (errent->xact_error); + + PG_RETURN_INT64(result); +} + +Datum +pg_stat_get_subscription_xact_error_bytes(PG_FUNCTION_ARGS) +{ + Oid subid = PG_GETARG_OID(0); + Oid relid; + int64 result; + PgStat_StatSubErrEntry *errent; + + if (PG_ARGISNULL(1)) + relid = InvalidOid; + else + relid = PG_GETARG_OID(1); + + /* Get subscription error entry */ + if ((errent = pgstat_fetch_subscription_error(subid, relid)) == NULL) + result = 0; + else + result = (int64) (errent->xact_error_bytes); + + PG_RETURN_INT64(result); +} + +Datum +pg_stat_get_subscription_xact_abort(PG_FUNCTION_ARGS) +{ + Oid subid = PG_GETARG_OID(0); + Oid relid; + int64 result; + PgStat_StatSubErrEntry *errent; + + if (PG_ARGISNULL(1)) + relid = InvalidOid; + else + relid = PG_GETARG_OID(1); + + /* Get subscription error entry */ + if ((errent = pgstat_fetch_subscription_error(subid, relid)) == NULL) + result = 0; + else + result = (int64) (errent->xact_abort); + + PG_RETURN_INT64(result); +} + +Datum +pg_stat_get_subscription_xact_abort_bytes(PG_FUNCTION_ARGS) +{ + Oid subid = PG_GETARG_OID(0); + Oid relid; + int64 result; + PgStat_StatSubErrEntry *errent; + + if (PG_ARGISNULL(1)) + relid = InvalidOid; + else + relid = PG_GETARG_OID(1); + + /* Get subscription error entry */ + if ((errent = pgstat_fetch_subscription_error(subid, relid)) == NULL) + result = 0; + else + result = (int64) (errent->xact_abort_bytes); + + PG_RETURN_INT64(result); +} diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index ea4e6d2..aeb935c 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -5393,6 +5393,30 @@ proargmodes => '{i,i,o,o,o,o,o,o,o,o,o,o}', proargnames => '{subid,relid,datid,subid,relid,command,xid,failure_source,failure_count,last_failure,last_failure_message,stats_reset}', prosrc => 'pg_stat_get_subscription_error' }, +{ oid => '8525', descr => 'statistics: number of transactions commit for a subscription', + proname => 'pg_stat_get_subscription_xact_commit', proisstrict => 'f', provolatile => 's', + proparallel => 'r', prorettype => 'int8', proargtypes => 'oid oid', + prosrc => 'pg_stat_get_subscription_xact_commit' }, +{ oid => '8526', descr => 'statistics: bytes of transactions commit for a subscription', + proname => 'pg_stat_get_subscription_xact_commit_bytes', proisstrict => 'f', provolatile => 's', + proparallel => 'r', prorettype => 'int8', proargtypes => 'oid oid', + prosrc => 'pg_stat_get_subscription_xact_commit_bytes' }, +{ oid => '8527', descr => 'statistics: number of transactions error for a subscription', + proname => 'pg_stat_get_subscription_xact_error', proisstrict => 'f', provolatile => 's', + proparallel => 'r', prorettype => 'int8', proargtypes => 'oid oid', + prosrc => 'pg_stat_get_subscription_xact_error' }, +{ oid => '8528', descr => 'statistics: bytes of transactions error for a subscription', + proname => 'pg_stat_get_subscription_xact_error_bytes', proisstrict => 'f', provolatile => 's', + proparallel => 'r', prorettype => 'int8', proargtypes => 'oid oid', + prosrc => 'pg_stat_get_subscription_xact_error_bytes' }, +{ oid => '8529', descr => 'statistics: number of transactions abort for a subscription', + proname => 'pg_stat_get_subscription_xact_abort', proisstrict => 'f', provolatile => 's', + proparallel => 'r', prorettype => 'int8', proargtypes => 'oid oid', + prosrc => 'pg_stat_get_subscription_xact_abort' }, +{ oid => '8530', descr => 'statistics: bytes of transactions abort for a subscription', + proname => 'pg_stat_get_subscription_xact_abort_bytes', proisstrict => 'f', provolatile => 's', + proparallel => 'r', prorettype => 'int8', proargtypes => 'oid oid', + prosrc => 'pg_stat_get_subscription_xact_abort_bytes' }, { oid => '6118', descr => 'statistics: information about subscription', proname => 'pg_stat_get_subscription', prorows => '10', proisstrict => 'f', proretset => 't', provolatile => 's', proparallel => 'r', diff --git a/src/include/pgstat.h b/src/include/pgstat.h index 6775736..2a43135 100644 --- a/src/include/pgstat.h +++ b/src/include/pgstat.h @@ -68,6 +68,7 @@ typedef enum StatMsgType PGSTAT_MTYPE_RESETSLRUCOUNTER, PGSTAT_MTYPE_RESETREPLSLOTCOUNTER, PGSTAT_MTYPE_SUBSCRIPTIONERR, + PGSTAT_MTYPE_SUBSCRIPTIONSUCCESS, PGSTAT_MTYPE_SUBSCRIPTIONERRPURGE, PGSTAT_MTYPE_SUBSCRIPTIONPURGE, PGSTAT_MTYPE_AUTOVAC_START, @@ -565,6 +566,7 @@ typedef struct PgStat_MsgSubscriptionErr Oid m_relid; LogicalRepMsgType m_command; TransactionId m_xid; + PgStat_Counter m_bytes; TimestampTz m_failure_time; char m_errmsg[PGSTAT_SUBSCRIPTIONERR_MSGLEN]; } PgStat_MsgSubscriptionErr; @@ -1002,8 +1004,12 @@ typedef struct PgStat_StatSubEntry * (subrelid is InvalidOid) or by the table sync worker (subrelid is a valid OID). * The error reported by the table sync worker is removed also when the table * synchronization process completed. + * + * Other general stats for transaction on this subscription are stored in this entry + * as well, aligned with transaction error and its total bytes. Holding the size of + * computed xact_error_bytes during replay depends on the apply error callback and + * having general stats in the same place reduces code complexity. */ - typedef struct PgStat_StatSubErrEntry { Oid subrelid; /* InvalidOid if the apply worker, otherwise @@ -1014,7 +1020,16 @@ typedef struct PgStat_StatSubErrEntry * case. */ LogicalRepMsgType command; TransactionId xid; - PgStat_Counter failure_count; + + PgStat_Counter xact_commit; + PgStat_Counter xact_commit_bytes; + PgStat_Counter xact_error; + PgStat_Counter xact_error_bytes; /* total error counts of this + subscription */ + PgStat_Counter xact_abort; + PgStat_Counter xact_abort_bytes; + PgStat_Counter failure_count; /* total error counts of one + specific error */ TimestampTz last_failure; char last_errmsg[PGSTAT_SUBSCRIPTIONERR_MSGLEN]; TimestampTz stat_reset_timestamp; @@ -1129,10 +1144,11 @@ extern void pgstat_report_checksum_failure(void); extern void pgstat_report_replslot(const PgStat_StatReplSlotEntry *repSlotStat); extern void pgstat_report_replslot_create(const char *slotname); extern void pgstat_report_replslot_drop(const char *slotname); +extern void pgstat_report_subscription_success(Oid subid, Oid subrel, + LogicalRepMsgType command, PgStat_Counter bytes); extern void pgstat_report_subscription_error(Oid subid, Oid subrelid, Oid relid, LogicalRepMsgType command, TransactionId xid, const char *errmsg); - extern void pgstat_initialize(void); diff --git a/src/include/replication/logicalworker.h b/src/include/replication/logicalworker.h index 2ad61a0..923c8ce 100644 --- a/src/include/replication/logicalworker.h +++ b/src/include/replication/logicalworker.h @@ -15,5 +15,9 @@ extern void ApplyWorkerMain(Datum main_arg); extern bool IsLogicalWorker(void); +extern int64 get_apply_error_context_xact_size(void); +extern void add_apply_error_context_xact_size(int64 size); +extern void reset_apply_error_context_xact_size(void); + #endif /* LOGICALWORKER_H */ diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out index 66b185f..89c68f1 100644 --- a/src/test/regress/expected/rules.out +++ b/src/test/regress/expected/rules.out @@ -2091,9 +2091,34 @@ pg_stat_subscription| SELECT su.oid AS subid, st.last_msg_send_time, st.last_msg_receipt_time, st.latest_end_lsn, + pg_stat_get_subscription_xact_commit(su.oid, st.relid) AS xact_commit, + pg_stat_get_subscription_xact_commit_bytes(su.oid, st.relid) AS xact_commit_bytes, + pg_stat_get_subscription_xact_error(su.oid, st.relid) AS xact_error, + pg_stat_get_subscription_xact_error_bytes(su.oid, st.relid) AS xact_error_bytes, + pg_stat_get_subscription_xact_abort(su.oid, st.relid) AS xact_abort, + pg_stat_get_subscription_xact_abort_bytes(su.oid, st.relid) AS xact_abort_bytes, st.latest_end_time FROM (pg_subscription su - LEFT JOIN pg_stat_get_subscription(NULL::oid) st(subid, relid, pid, received_lsn, last_msg_send_time, last_msg_receipt_time, latest_end_lsn, latest_end_time) ON ((st.subid = su.oid))); + LEFT JOIN ( SELECT s.subid, + s.relid, + s.pid, + s.received_lsn, + s.last_msg_send_time, + s.last_msg_receipt_time, + s.latest_end_lsn, + s.latest_end_time + FROM pg_stat_get_subscription(NULL::oid) s(subid, relid, pid, received_lsn, last_msg_send_time, last_msg_receipt_time, latest_end_lsn, latest_end_time) + UNION + SELECT r.srsubid, + r.srrelid, + NULL::integer AS pid, + NULL::pg_lsn AS received_lsn, + NULL::timestamp with time zone AS last_msg_send_time, + NULL::timestamp with time zone AS last_msg_receipt_time, + NULL::pg_lsn AS latest_end_lsn, + NULL::timestamp with time zone AS latest_end_time + FROM pg_subscription_rel r + WHERE (r.srsubstate <> 'r'::"char")) st ON ((st.subid = su.oid))); pg_stat_subscription_errors| SELECT d.datname, sr.subid, s.subname, ^ permalink raw reply [nested|flat] 113+ messages in thread
* RE: Failed transaction statistics to measure the logical replication progress @ 2021-09-09 08:03 [email protected] <[email protected]> parent: [email protected] <[email protected]> 0 siblings, 1 reply; 113+ messages in thread From: [email protected] @ 2021-09-09 08:03 UTC (permalink / raw) To: 'Amit Kapila' <[email protected]>; Masahiko Sawada <[email protected]>; +Cc: [email protected] <[email protected]> Hi On Thursday, September 2, 2021 11:23 AM I wrote: > I've made a new patch to extend pg_stat_subscription as suggested in [1] to > have columns xact_commit, xact_error and independent xact_abort mentioned > in [2]. > Also, during discussion, we touched a topic if we should include data sizes for > each column above and concluded that it's better to have ones. Accordingly, > I've implemented corresponding columns to show the data sizes as well. I've updated my previous patch of subscriber's stats. The main change of this version is the bytes calculation that are exported by pg_stat_subscription. I prepared a new function which is equivalent to ReorderBufferChangeSize on the subscriber side to calculate the resource consumption. It's because this is in line with actual process of the subscriber. Other changes are minor and cosmetic. Also, this patch is based on the v12 patch-set of skip xid, as described in my previous email. Note that you need to use past commit-id if you want to apply v12 set. I'll update mine as well when the latest patch-set v13 is shared on hackers. Best Regards, Takamichi Osumi Attachments: [application/octet-stream] extend_subscription_stats_of_transaction_v03.patch (32.4K, ../../TY2PR01MB489065DB7B8066936087C5E4EDD59@TY2PR01MB4890.jpnprd01.prod.outlook.com/2-extend_subscription_stats_of_transaction_v03.patch) download | inline diff: diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml index 0c02e46..1ce9f5a 100644 --- a/doc/src/sgml/monitoring.sgml +++ b/doc/src/sgml/monitoring.sgml @@ -3046,6 +3046,64 @@ SELECT pid, wait_event_type, wait_event FROM pg_stat_activity WHERE wait_event i <row> <entry role="catalog_table_entry"><para role="column_definition"> + <structfield>xact_commit</structfield> <type>bigint</type> + </para> + <para> + Number of transactions successfully applied in this subscription + </para></entry> + </row> + + <row> + <entry role="catalog_table_entry"><para role="column_definition"> + <structfield>xact_commit_bytes</structfield> <type>bigint</type> + </para> + <para> + Amount of transactions data successfully applied in this subscription + </para></entry> + </row> + + <row> + <entry role="catalog_table_entry"><para role="column_definition"> + <structfield>xact_error</structfield> <type>bigint</type> + </para> + <para> + Number of transactions unsuccessfully applied in this subscription + </para></entry> + </row> + + <row> + <entry role="catalog_table_entry"><para role="column_definition"> + <structfield>xact_error_bytes</structfield> <type>bigint</type> + </para> + <para> + Amount of transactions data unsuccessfully applied in this subscription + </para></entry> + </row> + + <row> + <entry role="catalog_table_entry"><para role="column_definition"> + <structfield>xact_abort</structfield> <type>bigint</type> + </para> + <para> + Number of transactions aborted in this subscription + </para></entry> + </row> + + <row> + <entry role="catalog_table_entry"><para role="column_definition"> + <structfield>xact_abort_bytes</structfield> <type>bigint</type> + </para> + <para> + Amount of transactions data aborted in this subscription. + Increase <literal>logical_decoding_work_mem</literal> on the publisher + to suppress unnecessary consumed network bandwidth or change in memory + of the subscriber, if unexpected amount of rollbacked transactions are + streamed. + </para></entry> + </row> + + <row> + <entry role="catalog_table_entry"><para role="column_definition"> <structfield>latest_end_time</structfield> <type>timestamp with time zone</type> </para> <para> diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index 449692a..109ba92 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -909,10 +909,37 @@ CREATE VIEW pg_stat_subscription AS st.last_msg_send_time, st.last_msg_receipt_time, st.latest_end_lsn, + pg_stat_get_subscription_xact_commit(su.oid, st.relid) AS xact_commit, + pg_stat_get_subscription_xact_commit_bytes(su.oid, st.relid) AS xact_commit_bytes, + pg_stat_get_subscription_xact_error(su.oid, st.relid) AS xact_error, + pg_stat_get_subscription_xact_error_bytes(su.oid, st.relid) AS xact_error_bytes, + pg_stat_get_subscription_xact_abort(su.oid, st.relid) AS xact_abort, + pg_stat_get_subscription_xact_abort_bytes(su.oid, st.relid) AS xact_abort_bytes, st.latest_end_time FROM pg_subscription su - LEFT JOIN pg_stat_get_subscription(NULL) st - ON (st.subid = su.oid); + LEFT JOIN + (SELECT + s.subid, + s.relid, + s.pid, + s.received_lsn, + s.last_msg_send_time, + s.last_msg_receipt_time, + s.latest_end_lsn, + s.latest_end_time + FROM pg_stat_get_subscription(NULL) s + UNION -- acquire relid of table sync worker + SELECT + r.srsubid, + r.srrelid, + NULL as pid, + NULL as received_lsn, + NULL as last_msg_send_time, + NULL as last_msg_receipt_time, + NULL as latest_end_lsn, + NULL as latest_end_time + FROM pg_subscription_rel r WHERE r.srsubstate <> 'r') st + ON (st.subid = su.oid); CREATE VIEW pg_stat_ssl AS SELECT diff --git a/src/backend/executor/execPartition.c b/src/backend/executor/execPartition.c index 5c723bc..79a25bd 100644 --- a/src/backend/executor/execPartition.c +++ b/src/backend/executor/execPartition.c @@ -194,6 +194,17 @@ static void find_matching_subplans_recurse(PartitionPruningData *prunedata, /* + * PartitionTupleRoutingSize - exported to calculate total data size + * of logical replication mesage apply, since it is one of the + * ApplyExecutionData struct members. + */ +size_t +PartitionTupleRoutingSize(void) +{ + return sizeof(PartitionTupleRouting); +} + +/* * ExecSetupPartitionTupleRouting - sets up information needed during * tuple routing for partitioned tables, encapsulates it in * PartitionTupleRouting, and returns it. diff --git a/src/backend/postmaster/pgstat.c b/src/backend/postmaster/pgstat.c index 778e409..bca3c41 100644 --- a/src/backend/postmaster/pgstat.c +++ b/src/backend/postmaster/pgstat.c @@ -55,6 +55,7 @@ #include "postmaster/postmaster.h" #include "replication/slot.h" #include "replication/walsender.h" +#include "replication/logicalworker.h" #include "storage/backendid.h" #include "storage/dsm.h" #include "storage/fd.h" @@ -378,6 +379,7 @@ static void pgstat_recv_checksum_failure(PgStat_MsgChecksumFailure *msg, int len static void pgstat_recv_connstat(PgStat_MsgConn *msg, int len); static void pgstat_recv_replslot(PgStat_MsgReplSlot *msg, int len); static void pgstat_recv_tempfile(PgStat_MsgTempFile *msg, int len); +static void pgstat_recv_subscription_success(PgStat_MsgSubscriptionErr *msg, int len); static void pgstat_recv_subscription_error(PgStat_MsgSubscriptionErr *msg, int len); static void pgstat_recv_subscription_error_purge(PgStat_MsgSubscriptionErrPurge *msg, int len); @@ -2038,6 +2040,40 @@ pgstat_report_replslot_drop(const char *slotname) } /* ---------- + * pgstat_report_subscription_success() - + * + * Tell the collector about the subscription success. + * ---------- + */ +void +pgstat_report_subscription_success(Oid subid, Oid subrel, + LogicalRepMsgType command, PgStat_Counter bytes) +{ + PgStat_MsgSubscriptionErr msg; + + Assert(command == 0 /* table sync worker */ || + command == LOGICAL_REP_MSG_COMMIT || + command == LOGICAL_REP_MSG_COMMIT_PREPARED || + command == LOGICAL_REP_MSG_STREAM_ABORT); + + pgstat_setheader(&msg.m_hdr, PGSTAT_MTYPE_SUBSCRIPTIONSUCCESS); + msg.m_subid = subid; + msg.m_subrelid = subrel; + msg.m_clear = false; + msg.m_reset = false; + msg.m_databaseid = InvalidOid; + msg.m_relid = InvalidOid; + msg.m_command = command; + msg.m_xid = InvalidTransactionId; + msg.m_bytes = bytes; + msg.m_failure_time = GetCurrentTimestamp(); + msg.m_errmsg[0] = '\0'; + pgstat_send(&msg, sizeof(PgStat_MsgSubscriptionErr)); + + reset_apply_error_context_xact_size(); +} + +/* ---------- * pgstat_report_subscription_error() - * * Tell the collector about the subscription error. @@ -2063,10 +2099,13 @@ pgstat_report_subscription_error(Oid subid, Oid subrelid, Oid relid, msg.m_relid = relid; msg.m_command = command; msg.m_xid = xid; + msg.m_bytes = get_apply_error_context_xact_size(); msg.m_failure_time = GetCurrentTimestamp(); strlcpy(msg.m_errmsg, errmsg, PGSTAT_SUBSCRIPTIONERR_MSGLEN); pgstat_send(&msg, len); + + reset_apply_error_context_xact_size(); } /* ---------- @@ -3730,6 +3769,10 @@ PgstatCollectorMain(int argc, char *argv[]) pgstat_recv_connstat(&msg.msg_conn, len); break; + case PGSTAT_MTYPE_SUBSCRIPTIONSUCCESS: + pgstat_recv_subscription_success(&msg.msg_subscriptionerr, len); + break; + case PGSTAT_MTYPE_SUBSCRIPTIONERR: pgstat_recv_subscription_error(&msg.msg_subscriptionerr, len); break; @@ -6153,6 +6196,45 @@ pgstat_recv_funcpurge(PgStat_MsgFuncpurge *msg, int len) } /* ---------- + * pgstat_recv_subscription_success() - + * + * Process a SUBSCRIPTIONERR message. + * ---------- + */ +static void +pgstat_recv_subscription_success(PgStat_MsgSubscriptionErr *msg, int len) +{ + PgStat_StatSubErrEntry *errent; + bool create = !(msg->m_reset || msg->m_clear); + + errent = pgstat_get_subscription_error_entry(msg->m_subid, + msg->m_subrelid, + create); + + /* msg from table sync worker */ + if (msg->m_command == 0) + errent->xact_commit++; + else + { + switch(msg->m_command) + { + case LOGICAL_REP_MSG_COMMIT: + case LOGICAL_REP_MSG_COMMIT_PREPARED: + errent->xact_commit++; + errent->xact_commit_bytes += msg->m_bytes; + break; + case LOGICAL_REP_MSG_STREAM_ABORT: + errent->xact_abort++; + errent->xact_abort_bytes = msg->m_bytes; + break; + default: + elog(ERROR, "unexpected command type"); + } + } + errent->last_failure = msg->m_failure_time; +} + +/* ---------- * pgstat_recv_subscription_error() - * * Process a SUBSCRIPTIONERR message. @@ -6202,6 +6284,8 @@ pgstat_recv_subscription_error(PgStat_MsgSubscriptionErr *msg, int len) errent->relid = msg->m_relid; errent->command = msg->m_command; errent->xid = msg->m_xid; + errent->xact_error++; + errent->xact_error_bytes += msg->m_bytes; errent->failure_count++; errent->last_failure = msg->m_failure_time; strlcpy(errent->last_errmsg, msg->m_errmsg, PGSTAT_SUBSCRIPTIONERR_MSGLEN); @@ -6465,6 +6549,12 @@ pgstat_get_subscription_error_entry(Oid subid, Oid subrelid, bool create) errent->relid = InvalidOid; errent->command = 0; errent->xid = InvalidTransactionId; + errent->xact_commit = 0; + errent->xact_commit_bytes = 0; + errent->xact_error = 0; + errent->xact_error_bytes = 0; + errent->xact_abort = 0; + errent->xact_abort_bytes = 0; errent->failure_count = 0; errent->last_failure = 0; errent->last_errmsg[0] = '\0'; diff --git a/src/backend/replication/logical/proto.c b/src/backend/replication/logical/proto.c index 9f5bf4b..2e1fc94 100644 --- a/src/backend/replication/logical/proto.c +++ b/src/backend/replication/logical/proto.c @@ -17,6 +17,7 @@ #include "catalog/pg_type.h" #include "libpq/pqformat.h" #include "replication/logicalproto.h" +#include "replication/logicalworker.h" #include "utils/lsyscache.h" #include "utils/syscache.h" diff --git a/src/backend/replication/logical/tablesync.c b/src/backend/replication/logical/tablesync.c index f07983a..7ba7486 100644 --- a/src/backend/replication/logical/tablesync.c +++ b/src/backend/replication/logical/tablesync.c @@ -1150,6 +1150,15 @@ copy_table_done: SpinLockRelease(&MyLogicalRepWorker->relmutex); /* + * Update the stats for table sync. We don't record the bytes of + * table synchronization. + */ + pgstat_report_subscription_success(MyLogicalRepWorker->subid, + MyLogicalRepWorker->relid, + 0, /* no corresponding logical message type */ + 0); + + /* * Finally, wait until the main apply worker tells us to catch up and then * return to let LogicalRepApplyLoop do it. */ diff --git a/src/backend/replication/logical/worker.c b/src/backend/replication/logical/worker.c index 34ed8e4..7eca5df 100644 --- a/src/backend/replication/logical/worker.c +++ b/src/backend/replication/logical/worker.c @@ -222,6 +222,20 @@ typedef struct ApplyErrorCallbackArg LogicalRepMsgType command; /* 0 if invalid */ LogicalRepRelMapEntry *rel; + /* + * Store data size of this transaction. + * + * The byte size of transaction on the publisher is calculated + * by ReorderBufferChangeSize() based on the ReorderBufferChange + * structure. But on the subscriber, consumed resources are + * not same as the publisher's decoding process and necessary + * to compute those in different way. Then, the exact same byte + * size is not restored on the subscriber usually. Further, in + * order to give accurate bytes even in the case of error, add + * byte size to this value step by step. + */ + PgStat_Counter bytes; + /* Remote node information */ int remote_attnum; /* -1 if invalid */ TransactionId remote_xid; @@ -232,6 +246,7 @@ static ApplyErrorCallbackArg apply_error_callback_arg = { .command = 0, .rel = NULL, + .bytes = 0, .remote_attnum = -1, .remote_xid = InvalidTransactionId, .ts = 0, @@ -328,6 +343,7 @@ static void maybe_reread_subscription(void); /* prototype needed because of stream_commit */ static void apply_dispatch(StringInfo s); +void update_apply_change_size(LogicalRepMsgType action, void *data); static void apply_handle_commit_internal(LogicalRepCommitData *commit_data); static void apply_handle_insert_internal(ApplyExecutionData *edata, @@ -863,6 +879,10 @@ apply_handle_commit(StringInfo s) process_syncing_tables(commit_data.end_lsn); pgstat_report_activity(STATE_IDLE, NULL); + pgstat_report_subscription_success(MySubscription->oid, + InvalidOid, + LOGICAL_REP_MSG_COMMIT, + apply_error_callback_arg.bytes); reset_apply_error_context_info(); } @@ -971,6 +991,9 @@ apply_handle_prepare(StringInfo s) in_remote_transaction = false; + /* Update the size of change in memory of this transaction */ + update_apply_change_size(LOGICAL_REP_MSG_PREPARE, NULL); + /* Process any tables that are being synchronized in parallel. */ process_syncing_tables(prepare_data.end_lsn); @@ -1012,10 +1035,17 @@ apply_handle_commit_prepared(StringInfo s) store_flush_position(prepare_data.end_lsn); in_remote_transaction = false; + /* Update the size of change in memory of this transaction */ + update_apply_change_size(LOGICAL_REP_MSG_COMMIT_PREPARED, NULL); + /* Process any tables that are being synchronized in parallel. */ process_syncing_tables(prepare_data.end_lsn); pgstat_report_activity(STATE_IDLE, NULL); + pgstat_report_subscription_success(MySubscription->oid, + InvalidOid, + LOGICAL_REP_MSG_COMMIT_PREPARED, + apply_error_callback_arg.bytes); reset_apply_error_context_info(); } @@ -1198,6 +1228,10 @@ apply_handle_stream_start(StringInfo s) oldctx = MemoryContextSwitchTo(ApplyContext); stream_fileset = palloc(sizeof(FileSet)); + + /* Update the size of change in memory of this transaction */ + update_apply_change_size(LOGICAL_REP_MSG_STREAM_START, NULL); + FileSetInit(stream_fileset); MemoryContextSwitchTo(oldctx); @@ -1350,6 +1384,10 @@ apply_handle_stream_abort(StringInfo s) if (is_skipping_changes()) stop_skipping_changes(InvalidXLogRecPtr, 0); + pgstat_report_subscription_success(MySubscription->oid, + InvalidOid, + LOGICAL_REP_MSG_STREAM_ABORT, + apply_error_callback_arg.bytes); reset_apply_error_context_info(); } @@ -1498,6 +1536,10 @@ apply_handle_stream_commit(StringInfo s) stop_skipping_changes(commit_data.end_lsn, commit_data.committime); store_flush_position(commit_data.end_lsn); + + /* Update the size of change in memory of this transaction */ + update_apply_change_size(LOGICAL_REP_MSG_STREAM_COMMIT, NULL); + in_remote_transaction = false; } else @@ -1650,6 +1692,9 @@ apply_handle_insert(StringInfo s) slot_fill_defaults(rel, estate, remoteslot); MemoryContextSwitchTo(oldctx); + /* Update the size of change in memory of this transaction */ + update_apply_change_size(LOGICAL_REP_MSG_INSERT, rel); + /* For a partitioned table, insert the tuple into a partition. */ if (rel->localrel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE) apply_handle_tuple_routing(edata, @@ -1807,6 +1852,9 @@ apply_handle_update(StringInfo s) has_oldtup ? &oldtup : &newtup); MemoryContextSwitchTo(oldctx); + /* Update the size of change in memory of this transaction */ + update_apply_change_size(LOGICAL_REP_MSG_INSERT, rel); + /* For a partitioned table, apply update to correct partition. */ if (rel->localrel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE) apply_handle_tuple_routing(edata, @@ -1941,6 +1989,9 @@ apply_handle_delete(StringInfo s) slot_store_data(remoteslot, rel, &oldtup); MemoryContextSwitchTo(oldctx); + /* Update the size of change in memory of this transaction */ + update_apply_change_size(LOGICAL_REP_MSG_INSERT, rel); + /* For a partitioned table, apply delete to correct partition. */ if (rel->localrel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE) apply_handle_tuple_routing(edata, @@ -2503,6 +2554,133 @@ apply_dispatch(StringInfo s) } /* + * Subscriber side implementation equivalent to ReorderBufferChangeSize + * on the publisher. + * + * According to the logical replication message type, record major + * resource consumptions of this subscription for each message. + * At present, do not collect data from generic functions to keep + * code simplicity. Also, by adding multiple values at once, + * reduce function calls of this function itself. + * + * 'data' controls detail handling of data size calculation. + */ +void +update_apply_change_size(LogicalRepMsgType action, void *data) +{ + int64 size = 0; + LogicalRepRelMapEntry *relmapentry; + LogicalRepRelation *reprelation; + int *stream_write_len; + + /* + * In streaming mode, stream_write_change is called + * instead of immediate apply. List up the messages types + * that can be caught by handle_streamed_transaction and + * treat the write length as the size of transaction so + * that we can export it as part of pg_stat_subscription. + */ + if (in_streamed_transaction && + (action == LOGICAL_REP_MSG_RELATION || + action == LOGICAL_REP_MSG_TYPE || + action == LOGICAL_REP_MSG_INSERT || + action == LOGICAL_REP_MSG_UPDATE || + action == LOGICAL_REP_MSG_DELETE || + action == LOGICAL_REP_MSG_TRUNCATE)) + { + stream_write_len = (int *) data; + size += *stream_write_len; + add_apply_error_context_xact_size(size); + return; + } + + switch (action) + { + /* Follow the same order as in the apply_dispatch */ + case LOGICAL_REP_MSG_BEGIN: + case LOGICAL_REP_MSG_COMMIT: + break; + + case LOGICAL_REP_MSG_INSERT: + case LOGICAL_REP_MSG_UPDATE: + case LOGICAL_REP_MSG_DELETE: + Assert(data != NULL); + + /* + * Compute size based on ApplyExecutionData. + * The size of LogicalRepRelMapEntry can be skipped because + * it is obtained from hash_search in logicalrep_rel_open. + */ + size += sizeof(ApplyExecutionData) + sizeof(EState) + + sizeof(ResultRelInfo) + sizeof(ResultRelInfo); + + /* + * Add some extra size if the target relation is partitioned. + * PartitionTupleRouting isn't exported so call the function + * that returns its size instead. + */ + relmapentry = (LogicalRepRelMapEntry *) data; + if (relmapentry->localrel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE) + size += sizeof(ModifyTableState) + PartitionTupleRoutingSize(); + break; + + case LOGICAL_REP_MSG_TRUNCATE: + /* No special consumption except for generic functions */ + break; + + case LOGICAL_REP_MSG_RELATION: + Assert(data != NULL); + + reprelation = (LogicalRepRelation *) data; + /* See logicalrep_read_attrs() for the last two */ + size += sizeof(LogicalRepRelation) + + reprelation->natts * sizeof(char *) + + reprelation->natts * sizeof(Oid); + break; + + case LOGICAL_REP_MSG_TYPE: + case LOGICAL_REP_MSG_ORIGIN: + case LOGICAL_REP_MSG_MESSAGE: + break; + + case LOGICAL_REP_MSG_STREAM_START: + size += sizeof(FileSet); + break; + + case LOGICAL_REP_MSG_STREAM_STOP: + case LOGICAL_REP_MSG_STREAM_ABORT: + break; + + case LOGICAL_REP_MSG_STREAM_COMMIT: + size += sizeof(FlushPosition); + break; + + case LOGICAL_REP_MSG_BEGIN_PREPARE: + case LOGICAL_REP_MSG_PREPARE: + break; + + case LOGICAL_REP_MSG_COMMIT_PREPARED: + size += sizeof(FlushPosition); + break; + + case LOGICAL_REP_MSG_ROLLBACK_PREPARED: + break; + + case LOGICAL_REP_MSG_STREAM_PREPARE: + size += sizeof(FlushPosition); + break; + + default: + ereport(ERROR, + (errcode(ERRCODE_PROTOCOL_VIOLATION), + errmsg("invalid logical replication message type \"%c\"", action))); + } + + /* update the total size of consumption */ + add_apply_error_context_xact_size(size); +} + +/* * Figure out which write/flush positions to report to the walsender process. * * We can't simply report back the last LSN the walsender sent us because the @@ -3362,6 +3540,7 @@ static void stream_write_change(char action, StringInfo s) { int len; + int total_len; Assert(in_streamed_transaction); Assert(TransactionIdIsValid(stream_xid)); @@ -3380,6 +3559,10 @@ stream_write_change(char action, StringInfo s) len = (s->len - s->cursor); BufFileWrite(stream_fd, &s->data[s->cursor], len); + + /* accumulate the total bytes processed in this function */ + total_len = (s->len - s->cursor) * 2 + sizeof(char) + sizeof(action); + update_apply_change_size(action, &total_len); } /* @@ -3744,6 +3927,27 @@ set_apply_error_context_xact(TransactionId xid, TimestampTz ts) apply_error_callback_arg.ts = ts; } +/* Exported so that stats collector can utilize this value */ +int64 +get_apply_error_context_xact_size(void) +{ + return apply_error_callback_arg.bytes; +} + +/* Add size to apply error callback bytes */ +void +add_apply_error_context_xact_size(int64 size) +{ + apply_error_callback_arg.bytes += size; +} + +/* Reset bytes information of apply error callback */ +void +reset_apply_error_context_xact_size(void) +{ + apply_error_callback_arg.bytes = 0; +} + /* Reset all information of apply error callback */ static inline void reset_apply_error_context_info(void) diff --git a/src/backend/utils/adt/pgstatfuncs.c b/src/backend/utils/adt/pgstatfuncs.c index c454e2f..0b4ad21 100644 --- a/src/backend/utils/adt/pgstatfuncs.c +++ b/src/backend/utils/adt/pgstatfuncs.c @@ -2492,3 +2492,136 @@ pg_stat_get_subscription_error(PG_FUNCTION_ARGS) /* Returns the record as Datum */ PG_RETURN_DATUM(HeapTupleGetDatum(heap_form_tuple(tupdesc, values, nulls))); } + +Datum +pg_stat_get_subscription_xact_commit(PG_FUNCTION_ARGS) +{ + Oid subid = PG_GETARG_OID(0); + Oid relid; + int64 result; + PgStat_StatSubErrEntry *errent; + + if (PG_ARGISNULL(1)) + relid = InvalidOid; + else + relid = PG_GETARG_OID(1); + + /* Get subscription error entry */ + if ((errent = pgstat_fetch_subscription_error(subid, relid)) == NULL) + result = 0; + else + result = (int64) (errent->xact_commit); + + PG_RETURN_INT64(result); +} + +Datum +pg_stat_get_subscription_xact_commit_bytes(PG_FUNCTION_ARGS) +{ + Oid subid = PG_GETARG_OID(0); + Oid relid; + int64 result; + PgStat_StatSubErrEntry *errent; + + if (PG_ARGISNULL(1)) + relid = InvalidOid; + else + relid = PG_GETARG_OID(1); + + /* Get subscription error entry */ + if ((errent = pgstat_fetch_subscription_error(subid, relid)) == NULL) + result = 0; + else + result = (int64) (errent->xact_commit_bytes); + + PG_RETURN_INT64(result); +} + + +Datum +pg_stat_get_subscription_xact_error(PG_FUNCTION_ARGS) +{ + Oid subid = PG_GETARG_OID(0); + Oid relid; + int64 result; + PgStat_StatSubErrEntry *errent; + + if (PG_ARGISNULL(1)) + relid = InvalidOid; + else + relid = PG_GETARG_OID(1); + + /* Get subscription error entry */ + if ((errent = pgstat_fetch_subscription_error(subid, relid)) == NULL) + result = 0; + else + result = (int64) (errent->xact_error); + + PG_RETURN_INT64(result); +} + +Datum +pg_stat_get_subscription_xact_error_bytes(PG_FUNCTION_ARGS) +{ + Oid subid = PG_GETARG_OID(0); + Oid relid; + int64 result; + PgStat_StatSubErrEntry *errent; + + if (PG_ARGISNULL(1)) + relid = InvalidOid; + else + relid = PG_GETARG_OID(1); + + /* Get subscription error entry */ + if ((errent = pgstat_fetch_subscription_error(subid, relid)) == NULL) + result = 0; + else + result = (int64) (errent->xact_error_bytes); + + PG_RETURN_INT64(result); +} + +Datum +pg_stat_get_subscription_xact_abort(PG_FUNCTION_ARGS) +{ + Oid subid = PG_GETARG_OID(0); + Oid relid; + int64 result; + PgStat_StatSubErrEntry *errent; + + if (PG_ARGISNULL(1)) + relid = InvalidOid; + else + relid = PG_GETARG_OID(1); + + /* Get subscription error entry */ + if ((errent = pgstat_fetch_subscription_error(subid, relid)) == NULL) + result = 0; + else + result = (int64) (errent->xact_abort); + + PG_RETURN_INT64(result); +} + +Datum +pg_stat_get_subscription_xact_abort_bytes(PG_FUNCTION_ARGS) +{ + Oid subid = PG_GETARG_OID(0); + Oid relid; + int64 result; + PgStat_StatSubErrEntry *errent; + + if (PG_ARGISNULL(1)) + relid = InvalidOid; + else + relid = PG_GETARG_OID(1); + + /* Get subscription error entry */ + if ((errent = pgstat_fetch_subscription_error(subid, relid)) == NULL) + result = 0; + else + result = (int64) (errent->xact_abort_bytes); + + PG_RETURN_INT64(result); +} diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index ea4e6d2..aeb935c 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -5393,6 +5393,30 @@ proargmodes => '{i,i,o,o,o,o,o,o,o,o,o,o}', proargnames => '{subid,relid,datid,subid,relid,command,xid,failure_source,failure_count,last_failure,last_failure_message,stats_reset}', prosrc => 'pg_stat_get_subscription_error' }, +{ oid => '8525', descr => 'statistics: number of transactions commit for a subscription', + proname => 'pg_stat_get_subscription_xact_commit', proisstrict => 'f', provolatile => 's', + proparallel => 'r', prorettype => 'int8', proargtypes => 'oid oid', + prosrc => 'pg_stat_get_subscription_xact_commit' }, +{ oid => '8526', descr => 'statistics: bytes of transactions commit for a subscription', + proname => 'pg_stat_get_subscription_xact_commit_bytes', proisstrict => 'f', provolatile => 's', + proparallel => 'r', prorettype => 'int8', proargtypes => 'oid oid', + prosrc => 'pg_stat_get_subscription_xact_commit_bytes' }, +{ oid => '8527', descr => 'statistics: number of transactions error for a subscription', + proname => 'pg_stat_get_subscription_xact_error', proisstrict => 'f', provolatile => 's', + proparallel => 'r', prorettype => 'int8', proargtypes => 'oid oid', + prosrc => 'pg_stat_get_subscription_xact_error' }, +{ oid => '8528', descr => 'statistics: bytes of transactions error for a subscription', + proname => 'pg_stat_get_subscription_xact_error_bytes', proisstrict => 'f', provolatile => 's', + proparallel => 'r', prorettype => 'int8', proargtypes => 'oid oid', + prosrc => 'pg_stat_get_subscription_xact_error_bytes' }, +{ oid => '8529', descr => 'statistics: number of transactions abort for a subscription', + proname => 'pg_stat_get_subscription_xact_abort', proisstrict => 'f', provolatile => 's', + proparallel => 'r', prorettype => 'int8', proargtypes => 'oid oid', + prosrc => 'pg_stat_get_subscription_xact_abort' }, +{ oid => '8530', descr => 'statistics: bytes of transactions abort for a subscription', + proname => 'pg_stat_get_subscription_xact_abort_bytes', proisstrict => 'f', provolatile => 's', + proparallel => 'r', prorettype => 'int8', proargtypes => 'oid oid', + prosrc => 'pg_stat_get_subscription_xact_abort_bytes' }, { oid => '6118', descr => 'statistics: information about subscription', proname => 'pg_stat_get_subscription', prorows => '10', proisstrict => 'f', proretset => 't', provolatile => 's', proparallel => 'r', diff --git a/src/include/executor/execPartition.h b/src/include/executor/execPartition.h index 694e38b..773e46c 100644 --- a/src/include/executor/execPartition.h +++ b/src/include/executor/execPartition.h @@ -110,6 +110,7 @@ typedef struct PartitionPruneState PartitionPruningData *partprunedata[FLEXIBLE_ARRAY_MEMBER]; } PartitionPruneState; +extern size_t PartitionTupleRoutingSize(void); extern PartitionTupleRouting *ExecSetupPartitionTupleRouting(EState *estate, Relation rel); extern ResultRelInfo *ExecFindPartition(ModifyTableState *mtstate, diff --git a/src/include/pgstat.h b/src/include/pgstat.h index 6775736..7fa53ed 100644 --- a/src/include/pgstat.h +++ b/src/include/pgstat.h @@ -67,6 +67,7 @@ typedef enum StatMsgType PGSTAT_MTYPE_RESETSINGLECOUNTER, PGSTAT_MTYPE_RESETSLRUCOUNTER, PGSTAT_MTYPE_RESETREPLSLOTCOUNTER, + PGSTAT_MTYPE_SUBSCRIPTIONSUCCESS, PGSTAT_MTYPE_SUBSCRIPTIONERR, PGSTAT_MTYPE_SUBSCRIPTIONERRPURGE, PGSTAT_MTYPE_SUBSCRIPTIONPURGE, @@ -565,6 +566,7 @@ typedef struct PgStat_MsgSubscriptionErr Oid m_relid; LogicalRepMsgType m_command; TransactionId m_xid; + PgStat_Counter m_bytes; TimestampTz m_failure_time; char m_errmsg[PGSTAT_SUBSCRIPTIONERR_MSGLEN]; } PgStat_MsgSubscriptionErr; @@ -1002,6 +1004,11 @@ typedef struct PgStat_StatSubEntry * (subrelid is InvalidOid) or by the table sync worker (subrelid is a valid OID). * The error reported by the table sync worker is removed also when the table * synchronization process completed. + * + * Other general stats for transaction on subscription are stored in this entry + * as well, aligned with transaction error and its total bytes. Holding the size of + * computed xact_error_bytes during replay depends on the apply error callback and + * having general stats in the same place reduces code complexity. */ typedef struct PgStat_StatSubErrEntry @@ -1014,7 +1021,18 @@ typedef struct PgStat_StatSubErrEntry * case. */ LogicalRepMsgType command; TransactionId xid; - PgStat_Counter failure_count; + + /* transaction stats of subscription */ + PgStat_Counter xact_commit; + PgStat_Counter xact_commit_bytes; + PgStat_Counter xact_error; + PgStat_Counter xact_error_bytes; /* total error counts of this + subscription */ + PgStat_Counter xact_abort; + PgStat_Counter xact_abort_bytes; + + PgStat_Counter failure_count; /* total error counts of one + specific error */ TimestampTz last_failure; char last_errmsg[PGSTAT_SUBSCRIPTIONERR_MSGLEN]; TimestampTz stat_reset_timestamp; @@ -1129,10 +1147,11 @@ extern void pgstat_report_checksum_failure(void); extern void pgstat_report_replslot(const PgStat_StatReplSlotEntry *repSlotStat); extern void pgstat_report_replslot_create(const char *slotname); extern void pgstat_report_replslot_drop(const char *slotname); +extern void pgstat_report_subscription_success(Oid subid, Oid subrel, + LogicalRepMsgType command, PgStat_Counter bytes); extern void pgstat_report_subscription_error(Oid subid, Oid subrelid, Oid relid, LogicalRepMsgType command, TransactionId xid, const char *errmsg); - extern void pgstat_initialize(void); diff --git a/src/include/replication/logicalworker.h b/src/include/replication/logicalworker.h index 2ad61a0..923c8ce 100644 --- a/src/include/replication/logicalworker.h +++ b/src/include/replication/logicalworker.h @@ -15,5 +15,9 @@ extern void ApplyWorkerMain(Datum main_arg); extern bool IsLogicalWorker(void); +extern int64 get_apply_error_context_xact_size(void); +extern void add_apply_error_context_xact_size(int64 size); +extern void reset_apply_error_context_xact_size(void); + #endif /* LOGICALWORKER_H */ diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out index 66b185f..89c68f1 100644 --- a/src/test/regress/expected/rules.out +++ b/src/test/regress/expected/rules.out @@ -2091,9 +2091,34 @@ pg_stat_subscription| SELECT su.oid AS subid, st.last_msg_send_time, st.last_msg_receipt_time, st.latest_end_lsn, + pg_stat_get_subscription_xact_commit(su.oid, st.relid) AS xact_commit, + pg_stat_get_subscription_xact_commit_bytes(su.oid, st.relid) AS xact_commit_bytes, + pg_stat_get_subscription_xact_error(su.oid, st.relid) AS xact_error, + pg_stat_get_subscription_xact_error_bytes(su.oid, st.relid) AS xact_error_bytes, + pg_stat_get_subscription_xact_abort(su.oid, st.relid) AS xact_abort, + pg_stat_get_subscription_xact_abort_bytes(su.oid, st.relid) AS xact_abort_bytes, st.latest_end_time FROM (pg_subscription su - LEFT JOIN pg_stat_get_subscription(NULL::oid) st(subid, relid, pid, received_lsn, last_msg_send_time, last_msg_receipt_time, latest_end_lsn, latest_end_time) ON ((st.subid = su.oid))); + LEFT JOIN ( SELECT s.subid, + s.relid, + s.pid, + s.received_lsn, + s.last_msg_send_time, + s.last_msg_receipt_time, + s.latest_end_lsn, + s.latest_end_time + FROM pg_stat_get_subscription(NULL::oid) s(subid, relid, pid, received_lsn, last_msg_send_time, last_msg_receipt_time, latest_end_lsn, latest_end_time) + UNION + SELECT r.srsubid, + r.srrelid, + NULL::integer AS pid, + NULL::pg_lsn AS received_lsn, + NULL::timestamp with time zone AS last_msg_send_time, + NULL::timestamp with time zone AS last_msg_receipt_time, + NULL::pg_lsn AS latest_end_lsn, + NULL::timestamp with time zone AS latest_end_time + FROM pg_subscription_rel r + WHERE (r.srsubstate <> 'r'::"char")) st ON ((st.subid = su.oid))); pg_stat_subscription_errors| SELECT d.datname, sr.subid, s.subname, ^ permalink raw reply [nested|flat] 113+ messages in thread
* RE: Failed transaction statistics to measure the logical replication progress @ 2021-09-14 03:11 [email protected] <[email protected]> parent: [email protected] <[email protected]> 0 siblings, 1 reply; 113+ messages in thread From: [email protected] @ 2021-09-14 03:11 UTC (permalink / raw) To: 'Amit Kapila' <[email protected]>; Masahiko Sawada <[email protected]>; +Cc: [email protected] <[email protected]> Hi On Thursday, September 9, 2021 5:04 PM I wrote: > Also, this patch is based on the v12 patch-set of skip xid, as described in my > previous email. > Note that you need to use past commit-id if you want to apply v12 set. > I'll update mine as well when the latest patch-set v13 is shared on hackers. Rebased, using v13 patch-set in [1]. [1 ] - https://www.postgresql.org/message-id/CAD21AoBUXM4ODfPFa%3Dh7M6vSKwOKysapUce3tS4rs9mfVMm%2BcQ%40mail... Best Regards, Takamichi Osumi Attachments: [application/octet-stream] extend_subscription_stats_of_transaction_v04.patch (31.8K, ../../OSBPR01MB4888507DD52192F0CDF9A49DEDDA9@OSBPR01MB4888.jpnprd01.prod.outlook.com/2-extend_subscription_stats_of_transaction_v04.patch) download | inline diff: diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml index 3a4c98b..8bd44af 100644 --- a/doc/src/sgml/monitoring.sgml +++ b/doc/src/sgml/monitoring.sgml @@ -3046,6 +3046,64 @@ SELECT pid, wait_event_type, wait_event FROM pg_stat_activity WHERE wait_event i <row> <entry role="catalog_table_entry"><para role="column_definition"> + <structfield>xact_commit</structfield> <type>bigint</type> + </para> + <para> + Number of transactions successfully applied in this subscription + </para></entry> + </row> + + <row> + <entry role="catalog_table_entry"><para role="column_definition"> + <structfield>xact_commit_bytes</structfield> <type>bigint</type> + </para> + <para> + Amount of transactions data successfully applied in this subscription + </para></entry> + </row> + + <row> + <entry role="catalog_table_entry"><para role="column_definition"> + <structfield>xact_error</structfield> <type>bigint</type> + </para> + <para> + Number of transactions unsuccessfully applied in this subscription + </para></entry> + </row> + + <row> + <entry role="catalog_table_entry"><para role="column_definition"> + <structfield>xact_error_bytes</structfield> <type>bigint</type> + </para> + <para> + Amount of transactions data unsuccessfully applied in this subscription + </para></entry> + </row> + + <row> + <entry role="catalog_table_entry"><para role="column_definition"> + <structfield>xact_abort</structfield> <type>bigint</type> + </para> + <para> + Number of transactions aborted in this subscription + </para></entry> + </row> + + <row> + <entry role="catalog_table_entry"><para role="column_definition"> + <structfield>xact_abort_bytes</structfield> <type>bigint</type> + </para> + <para> + Amount of transactions data aborted in this subscription. + Increase <literal>logical_decoding_work_mem</literal> on the publisher + to suppress unnecessary consumed network bandwidth or change in memory + of the subscriber, if unexpected amount of rollbacked transactions are + streamed. + </para></entry> + </row> + + <row> + <entry role="catalog_table_entry"><para role="column_definition"> <structfield>latest_end_time</structfield> <type>timestamp with time zone</type> </para> <para> diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index b0cd8d2..3a2c69b 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -909,10 +909,37 @@ CREATE VIEW pg_stat_subscription AS st.last_msg_send_time, st.last_msg_receipt_time, st.latest_end_lsn, + pg_stat_get_subscription_xact_commit(su.oid, st.relid) AS xact_commit, + pg_stat_get_subscription_xact_commit_bytes(su.oid, st.relid) AS xact_commit_bytes, + pg_stat_get_subscription_xact_error(su.oid, st.relid) AS xact_error, + pg_stat_get_subscription_xact_error_bytes(su.oid, st.relid) AS xact_error_bytes, + pg_stat_get_subscription_xact_abort(su.oid, st.relid) AS xact_abort, + pg_stat_get_subscription_xact_abort_bytes(su.oid, st.relid) AS xact_abort_bytes, st.latest_end_time FROM pg_subscription su - LEFT JOIN pg_stat_get_subscription(NULL) st - ON (st.subid = su.oid); + LEFT JOIN + (SELECT + s.subid, + s.relid, + s.pid, + s.received_lsn, + s.last_msg_send_time, + s.last_msg_receipt_time, + s.latest_end_lsn, + s.latest_end_time + FROM pg_stat_get_subscription(NULL) s + UNION -- acquire relid of table sync worker + SELECT + r.srsubid, + r.srrelid, + NULL as pid, + NULL as received_lsn, + NULL as last_msg_send_time, + NULL as last_msg_receipt_time, + NULL as latest_end_lsn, + NULL as latest_end_time + FROM pg_subscription_rel r WHERE r.srsubstate <> 'r') st + ON (st.subid = su.oid); CREATE VIEW pg_stat_ssl AS SELECT diff --git a/src/backend/executor/execPartition.c b/src/backend/executor/execPartition.c index 5c723bc..107a7aa 100644 --- a/src/backend/executor/execPartition.c +++ b/src/backend/executor/execPartition.c @@ -194,6 +194,17 @@ static void find_matching_subplans_recurse(PartitionPruningData *prunedata, /* + * PartitionTupleRoutingSize - exported to calculate total data size + * of logical replication mesage apply, because this is one of the + * ApplyExecutionData struct members. + */ +size_t +PartitionTupleRoutingSize(void) +{ + return sizeof(PartitionTupleRouting); +} + +/* * ExecSetupPartitionTupleRouting - sets up information needed during * tuple routing for partitioned tables, encapsulates it in * PartitionTupleRouting, and returns it. diff --git a/src/backend/postmaster/pgstat.c b/src/backend/postmaster/pgstat.c index da4c493..c91ecf3 100644 --- a/src/backend/postmaster/pgstat.c +++ b/src/backend/postmaster/pgstat.c @@ -56,6 +56,7 @@ #include "postmaster/postmaster.h" #include "replication/slot.h" #include "replication/walsender.h" +#include "replication/logicalworker.h" #include "storage/backendid.h" #include "storage/dsm.h" #include "storage/fd.h" @@ -381,6 +382,7 @@ static void pgstat_recv_checksum_failure(PgStat_MsgChecksumFailure *msg, int len static void pgstat_recv_connstat(PgStat_MsgConn *msg, int len); static void pgstat_recv_replslot(PgStat_MsgReplSlot *msg, int len); static void pgstat_recv_tempfile(PgStat_MsgTempFile *msg, int len); +static void pgstat_recv_subscription_success(PgStat_MsgSubscriptionErr *msg, int len); static void pgstat_recv_subscription_error(PgStat_MsgSubscriptionErr *msg, int len); static void pgstat_recv_subscription_error_purge(PgStat_MsgSubscriptionErrPurge *msg, int len); @@ -2039,6 +2041,39 @@ pgstat_report_replslot_drop(const char *slotname) } /* ---------- + * pgstat_report_subscription_success() - + * + * Tell the collector about the subscription success. + * ---------- + */ +void +pgstat_report_subscription_success(Oid subid, Oid subrel, + LogicalRepMsgType command, PgStat_Counter bytes) +{ + PgStat_MsgSubscriptionErr msg; + + Assert(command == 0 /* table sync worker */ || + command == LOGICAL_REP_MSG_COMMIT || + command == LOGICAL_REP_MSG_COMMIT_PREPARED || + command == LOGICAL_REP_MSG_STREAM_ABORT); + + pgstat_setheader(&msg.m_hdr, PGSTAT_MTYPE_SUBSCRIPTIONSUCCESS); + msg.m_subid = subid; + msg.m_subrelid = subrel; + msg.m_clear = false; + msg.m_reset = false; + msg.m_relid = InvalidOid; + msg.m_command = command; + msg.m_xid = InvalidTransactionId; + msg.m_bytes = bytes; + msg.m_failure_time = GetCurrentTimestamp(); + msg.m_errmsg[0] = '\0'; + pgstat_send(&msg, sizeof(PgStat_MsgSubscriptionErr)); + + reset_apply_error_context_xact_size(); +} + +/* ---------- * pgstat_report_subscription_error() - * * Tell the collector about the subscription error. @@ -2063,10 +2098,13 @@ pgstat_report_subscription_error(Oid subid, Oid subrelid, Oid relid, msg.m_relid = relid; msg.m_command = command; msg.m_xid = xid; + msg.m_bytes = get_apply_error_context_xact_size(); msg.m_failure_time = GetCurrentTimestamp(); strlcpy(msg.m_errmsg, errmsg, PGSTAT_SUBSCRIPTIONERR_MSGLEN); pgstat_send(&msg, len); + + reset_apply_error_context_xact_size(); } /* ---------- @@ -3730,6 +3768,10 @@ PgstatCollectorMain(int argc, char *argv[]) pgstat_recv_connstat(&msg.msg_conn, len); break; + case PGSTAT_MTYPE_SUBSCRIPTIONSUCCESS: + pgstat_recv_subscription_success(&msg.msg_subscriptionerr, len); + break; + case PGSTAT_MTYPE_SUBSCRIPTIONERR: pgstat_recv_subscription_error(&msg.msg_subscriptionerr, len); break; @@ -6172,6 +6214,45 @@ pgstat_recv_funcpurge(PgStat_MsgFuncpurge *msg, int len) } /* ---------- + * pgstat_recv_subscription_success() - + * + * Process a SUBSCRIPTIONERR message. + * ---------- + */ +static void +pgstat_recv_subscription_success(PgStat_MsgSubscriptionErr *msg, int len) +{ + PgStat_StatSubErrEntry *errent; + bool create = !(msg->m_reset || msg->m_clear); + + errent = pgstat_get_subscription_error_entry(msg->m_subid, + msg->m_subrelid, + create); + + /* msg from table sync worker */ + if (msg->m_command == 0) + errent->xact_commit++; + else + { + switch(msg->m_command) + { + case LOGICAL_REP_MSG_COMMIT: + case LOGICAL_REP_MSG_COMMIT_PREPARED: + errent->xact_commit++; + errent->xact_commit_bytes += msg->m_bytes; + break; + case LOGICAL_REP_MSG_STREAM_ABORT: + errent->xact_abort++; + errent->xact_abort_bytes = msg->m_bytes; + break; + default: + elog(ERROR, "unexpected command type"); + } + } + errent->last_failure = msg->m_failure_time; +} + +/* ---------- * pgstat_recv_subscription_error() - * * Process a SUBSCRIPTIONERR message. @@ -6232,6 +6313,8 @@ pgstat_recv_subscription_error(PgStat_MsgSubscriptionErr *msg, int len) /* update the error entry */ errent->command = msg->m_command; errent->xid = msg->m_xid; + errent->xact_error++; + errent->xact_error_bytes += msg->m_bytes; errent->failure_count++; errent->last_failure = msg->m_failure_time; strlcpy(errent->last_errmsg, msg->m_errmsg, PGSTAT_SUBSCRIPTIONERR_MSGLEN); @@ -6533,6 +6616,12 @@ pgstat_reset_subscription_error_entry(PgStat_StatSubErrEntry *errent, { errent->command = 0; errent->xid = InvalidTransactionId; + errent->xact_commit = 0; + errent->xact_commit_bytes = 0; + errent->xact_error = 0; + errent->xact_error_bytes = 0; + errent->xact_abort = 0; + errent->xact_abort_bytes = 0; errent->failure_count = 0; errent->last_failure = 0; errent->last_errmsg[0] = '\0'; diff --git a/src/backend/replication/logical/proto.c b/src/backend/replication/logical/proto.c index 9f5bf4b..2e1fc94 100644 --- a/src/backend/replication/logical/proto.c +++ b/src/backend/replication/logical/proto.c @@ -17,6 +17,7 @@ #include "catalog/pg_type.h" #include "libpq/pqformat.h" #include "replication/logicalproto.h" +#include "replication/logicalworker.h" #include "utils/lsyscache.h" #include "utils/syscache.h" diff --git a/src/backend/replication/logical/tablesync.c b/src/backend/replication/logical/tablesync.c index f07983a..7ba7486 100644 --- a/src/backend/replication/logical/tablesync.c +++ b/src/backend/replication/logical/tablesync.c @@ -1150,6 +1150,15 @@ copy_table_done: SpinLockRelease(&MyLogicalRepWorker->relmutex); /* + * Update the stats for table sync. We don't record the bytes of + * table synchronization. + */ + pgstat_report_subscription_success(MyLogicalRepWorker->subid, + MyLogicalRepWorker->relid, + 0, /* no corresponding logical message type */ + 0); + + /* * Finally, wait until the main apply worker tells us to catch up and then * return to let LogicalRepApplyLoop do it. */ diff --git a/src/backend/replication/logical/worker.c b/src/backend/replication/logical/worker.c index b20de59..1cd1d26 100644 --- a/src/backend/replication/logical/worker.c +++ b/src/backend/replication/logical/worker.c @@ -222,6 +222,20 @@ typedef struct ApplyErrorCallbackArg LogicalRepMsgType command; /* 0 if invalid */ LogicalRepRelMapEntry *rel; + /* + * Store data size of this transaction. + * + * The byte size of transaction on the publisher is calculated + * by ReorderBufferChangeSize() based on the ReorderBufferChange + * structure. But on the subscriber, consumed resources are + * not same as the publisher's decoding process and necessary + * to compute those in different way. Then, the exact same byte + * size is not restored on the subscriber usually. Further, in + * order to give accurate bytes even in the case of error, add + * byte size to this value step by step. + */ + PgStat_Counter bytes; + /* Remote node information */ int remote_attnum; /* -1 if invalid */ TransactionId remote_xid; @@ -232,6 +246,7 @@ static ApplyErrorCallbackArg apply_error_callback_arg = { .command = 0, .rel = NULL, + .bytes = 0, .remote_attnum = -1, .remote_xid = InvalidTransactionId, .ts = 0, @@ -319,6 +334,7 @@ static void maybe_reread_subscription(void); /* prototype needed because of stream_commit */ static void apply_dispatch(StringInfo s); +void update_apply_change_size(LogicalRepMsgType action, void *data); static void apply_handle_commit_internal(LogicalRepCommitData *commit_data); static void apply_handle_insert_internal(ApplyExecutionData *edata, @@ -852,6 +868,10 @@ apply_handle_commit(StringInfo s) /* Process any tables that are being synchronized in parallel. */ process_syncing_tables(commit_data.end_lsn); + pgstat_report_subscription_success(MySubscription->oid, + InvalidOid, + LOGICAL_REP_MSG_COMMIT, + apply_error_callback_arg.bytes); pgstat_report_activity(STATE_IDLE, NULL); reset_apply_error_context_info(); @@ -965,6 +985,7 @@ apply_handle_prepare(StringInfo s) /* Process any tables that are being synchronized in parallel. */ process_syncing_tables(prepare_data.end_lsn); + update_apply_change_size(LOGICAL_REP_MSG_PREPARE, NULL); pgstat_report_activity(STATE_IDLE, NULL); reset_apply_error_context_info(); } @@ -1006,6 +1027,12 @@ apply_handle_commit_prepared(StringInfo s) /* Process any tables that are being synchronized in parallel. */ process_syncing_tables(prepare_data.end_lsn); + update_apply_change_size(LOGICAL_REP_MSG_COMMIT_PREPARED, NULL); + pgstat_report_subscription_success(MySubscription->oid, + InvalidOid, + LOGICAL_REP_MSG_COMMIT_PREPARED, + apply_error_callback_arg.bytes); + pgstat_report_activity(STATE_IDLE, NULL); reset_apply_error_context_info(); } @@ -1193,6 +1220,7 @@ apply_handle_stream_start(StringInfo s) MyLogicalRepWorker->stream_fileset = palloc(sizeof(FileSet)); FileSetInit(MyLogicalRepWorker->stream_fileset); + update_apply_change_size(LOGICAL_REP_MSG_STREAM_START, NULL); MemoryContextSwitchTo(oldctx); } @@ -1345,6 +1373,10 @@ apply_handle_stream_abort(StringInfo s) if (is_skipping_changes()) stop_skipping_changes(InvalidXLogRecPtr, 0); + pgstat_report_subscription_success(MySubscription->oid, + InvalidOid, + LOGICAL_REP_MSG_STREAM_ABORT, + apply_error_callback_arg.bytes); reset_apply_error_context_info(); } @@ -1494,6 +1526,10 @@ apply_handle_stream_commit(StringInfo s) stop_skipping_changes(commit_data.end_lsn, commit_data.committime); store_flush_position(commit_data.end_lsn); + + /* Update the size of change in memory of this transaction */ + update_apply_change_size(LOGICAL_REP_MSG_STREAM_COMMIT, NULL); + in_remote_transaction = false; } else @@ -1646,6 +1682,8 @@ apply_handle_insert(StringInfo s) slot_fill_defaults(rel, estate, remoteslot); MemoryContextSwitchTo(oldctx); + update_apply_change_size(LOGICAL_REP_MSG_INSERT, rel); + /* For a partitioned table, insert the tuple into a partition. */ if (rel->localrel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE) apply_handle_tuple_routing(edata, @@ -1803,6 +1841,8 @@ apply_handle_update(StringInfo s) has_oldtup ? &oldtup : &newtup); MemoryContextSwitchTo(oldctx); + update_apply_change_size(LOGICAL_REP_MSG_UPDATE, rel); + /* For a partitioned table, apply update to correct partition. */ if (rel->localrel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE) apply_handle_tuple_routing(edata, @@ -1937,6 +1977,7 @@ apply_handle_delete(StringInfo s) slot_store_data(remoteslot, rel, &oldtup); MemoryContextSwitchTo(oldctx); + update_apply_change_size(LOGICAL_REP_MSG_DELETE, rel); /* For a partitioned table, apply delete to correct partition. */ if (rel->localrel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE) apply_handle_tuple_routing(edata, @@ -2498,6 +2539,133 @@ apply_dispatch(StringInfo s) apply_error_callback_arg.command = saved_command; } + +/* + * Subscriber side implementation equivalent to ReorderBufferChangeSize + * of the publisher. + * + * According to the logical replication message type, record major + * resource consumptions of this subscription for each message. + * At present, do not collect data from generic functions to keep + * code simplicity. Also, by adding multiple values at once, + * reduce number of function calls. + * + * 'data' controls detail handling of data size calculation. + */ +void +update_apply_change_size(LogicalRepMsgType action, void *data) +{ + int64 size = 0; + LogicalRepRelMapEntry *relmapentry; + LogicalRepRelation *reprelation; + int *stream_write_len; + + /* + * In streaming mode, stream_write_change is called + * instead of immediate apply. List up the messages types + * that can be caught by handle_streamed_transaction and + * treat the write length as the size of transaction so + * that we can export it as part of pg_stat_subscription. + */ + if (in_streamed_transaction && + (action == LOGICAL_REP_MSG_INSERT || + action == LOGICAL_REP_MSG_UPDATE || + action == LOGICAL_REP_MSG_DELETE || + action == LOGICAL_REP_MSG_TRUNCATE || + action == LOGICAL_REP_MSG_RELATION || + action == LOGICAL_REP_MSG_TYPE)) + { + stream_write_len = (int *) data; + size += *stream_write_len; + add_apply_error_context_xact_size(size); + return; + } + + switch (action) + { + /* Follow the same order as in the apply_dispatch */ + case LOGICAL_REP_MSG_BEGIN: + case LOGICAL_REP_MSG_COMMIT: + break; + + case LOGICAL_REP_MSG_INSERT: + case LOGICAL_REP_MSG_UPDATE: + case LOGICAL_REP_MSG_DELETE: + Assert(data != NULL); + + /* + * Compute size based on ApplyExecutionData. + * The size of LogicalRepRelMapEntry can be skipped because + * it is obtained from hash_search in logicalrep_rel_open. + */ + size += sizeof(ApplyExecutionData) + sizeof(EState) + + sizeof(ResultRelInfo) + sizeof(ResultRelInfo); + + /* + * Add some extra size if the target relation is partitioned. + * PartitionTupleRouting isn't exported. Therefore, call the + * function that returns its size instead. + */ + relmapentry = (LogicalRepRelMapEntry *) data; + if (relmapentry->localrel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE) + size += sizeof(ModifyTableState) + PartitionTupleRoutingSize(); + break; + + case LOGICAL_REP_MSG_TRUNCATE: + /* No special consumption except for generic functions */ + break; + + case LOGICAL_REP_MSG_RELATION: + Assert(data != NULL); + + reprelation = (LogicalRepRelation *) data; + /* See logicalrep_read_attrs for the last two */ + size += sizeof(LogicalRepRelation) + + reprelation->natts * sizeof(char *) + + reprelation->natts * sizeof(Oid); + break; + case LOGICAL_REP_MSG_TYPE: + case LOGICAL_REP_MSG_ORIGIN: + case LOGICAL_REP_MSG_MESSAGE: + break; + + case LOGICAL_REP_MSG_STREAM_START: + size += sizeof(FileSet); + break; + + case LOGICAL_REP_MSG_STREAM_STOP: + case LOGICAL_REP_MSG_STREAM_ABORT: + break; + + case LOGICAL_REP_MSG_STREAM_COMMIT: + size += sizeof(FlushPosition); + break; + + case LOGICAL_REP_MSG_BEGIN_PREPARE: + case LOGICAL_REP_MSG_PREPARE: + break; + + case LOGICAL_REP_MSG_COMMIT_PREPARED: + size += sizeof(FlushPosition); + break; + + case LOGICAL_REP_MSG_ROLLBACK_PREPARED: + break; + + case LOGICAL_REP_MSG_STREAM_PREPARE: + size += sizeof(FlushPosition); + break; + + default: + ereport(ERROR, + (errcode(ERRCODE_PROTOCOL_VIOLATION), + errmsg("invalid logical replication message type \"%c\"", action))); + } + /* update the total size of consumption */ + add_apply_error_context_xact_size(size); +} + + /* * Figure out which write/flush positions to report to the walsender process. * @@ -3352,6 +3520,7 @@ static void stream_write_change(char action, StringInfo s) { int len; + int total_len; Assert(in_streamed_transaction); Assert(TransactionIdIsValid(stream_xid)); @@ -3370,6 +3539,9 @@ stream_write_change(char action, StringInfo s) len = (s->len - s->cursor); BufFileWrite(stream_fd, &s->data[s->cursor], len); + + total_len = (s->len - s->cursor) * 2 + sizeof(char) + sizeof(action); + update_apply_change_size(action, &total_len); } /* @@ -3734,6 +3906,27 @@ set_apply_error_context_xact(TransactionId xid, TimestampTz ts) apply_error_callback_arg.ts = ts; } +/* Exported so that stats collector can utilize this value */ +int64 +get_apply_error_context_xact_size(void) +{ + return apply_error_callback_arg.bytes; +} + +/* Add size to apply error callback bytes */ +void +add_apply_error_context_xact_size(int64 size) +{ + apply_error_callback_arg.bytes += size; +} + +/* Reset bytes information of apply error callback */ +void +reset_apply_error_context_xact_size(void) +{ + apply_error_callback_arg.bytes = 0; +} + /* Reset all information of apply error callback */ static inline void reset_apply_error_context_info(void) diff --git a/src/backend/utils/adt/pgstatfuncs.c b/src/backend/utils/adt/pgstatfuncs.c index 51f693c..540de86 100644 --- a/src/backend/utils/adt/pgstatfuncs.c +++ b/src/backend/utils/adt/pgstatfuncs.c @@ -2491,3 +2491,136 @@ pg_stat_get_subscription_error(PG_FUNCTION_ARGS) /* Returns the record as Datum */ PG_RETURN_DATUM(HeapTupleGetDatum(heap_form_tuple(tupdesc, values, nulls))); } + +Datum +pg_stat_get_subscription_xact_commit(PG_FUNCTION_ARGS) +{ + Oid subid = PG_GETARG_OID(0); + Oid relid; + int64 result; + PgStat_StatSubErrEntry *errent; + + if (PG_ARGISNULL(1)) + relid = InvalidOid; + else + relid = PG_GETARG_OID(1); + + /* Get subscription error entry */ + if ((errent = pgstat_fetch_subscription_error(subid, relid)) == NULL) + result = 0; + else + result = (int64) (errent->xact_commit); + + PG_RETURN_INT64(result); +} + +Datum +pg_stat_get_subscription_xact_commit_bytes(PG_FUNCTION_ARGS) +{ + Oid subid = PG_GETARG_OID(0); + Oid relid; + int64 result; + PgStat_StatSubErrEntry *errent; + + if (PG_ARGISNULL(1)) + relid = InvalidOid; + else + relid = PG_GETARG_OID(1); + + /* Get subscription error entry */ + if ((errent = pgstat_fetch_subscription_error(subid, relid)) == NULL) + result = 0; + else + result = (int64) (errent->xact_commit_bytes); + + PG_RETURN_INT64(result); +} + + +Datum +pg_stat_get_subscription_xact_error(PG_FUNCTION_ARGS) +{ + Oid subid = PG_GETARG_OID(0); + Oid relid; + int64 result; + PgStat_StatSubErrEntry *errent; + + if (PG_ARGISNULL(1)) + relid = InvalidOid; + else + relid = PG_GETARG_OID(1); + + /* Get subscription error entry */ + if ((errent = pgstat_fetch_subscription_error(subid, relid)) == NULL) + result = 0; + else + result = (int64) (errent->xact_error); + + PG_RETURN_INT64(result); +} + +Datum +pg_stat_get_subscription_xact_error_bytes(PG_FUNCTION_ARGS) +{ + Oid subid = PG_GETARG_OID(0); + Oid relid; + int64 result; + PgStat_StatSubErrEntry *errent; + + if (PG_ARGISNULL(1)) + relid = InvalidOid; + else + relid = PG_GETARG_OID(1); + + /* Get subscription error entry */ + if ((errent = pgstat_fetch_subscription_error(subid, relid)) == NULL) + result = 0; + else + result = (int64) (errent->xact_error_bytes); + + PG_RETURN_INT64(result); +} + +Datum +pg_stat_get_subscription_xact_abort(PG_FUNCTION_ARGS) +{ + Oid subid = PG_GETARG_OID(0); + Oid relid; + int64 result; + PgStat_StatSubErrEntry *errent; + + if (PG_ARGISNULL(1)) + relid = InvalidOid; + else + relid = PG_GETARG_OID(1); + + /* Get subscription error entry */ + if ((errent = pgstat_fetch_subscription_error(subid, relid)) == NULL) + result = 0; + else + result = (int64) (errent->xact_abort); + + PG_RETURN_INT64(result); +} + +Datum +pg_stat_get_subscription_xact_abort_bytes(PG_FUNCTION_ARGS) +{ + Oid subid = PG_GETARG_OID(0); + Oid relid; + int64 result; + PgStat_StatSubErrEntry *errent; + + if (PG_ARGISNULL(1)) + relid = InvalidOid; + else + relid = PG_GETARG_OID(1); + + /* Get subscription error entry */ + if ((errent = pgstat_fetch_subscription_error(subid, relid)) == NULL) + result = 0; + else + result = (int64) (errent->xact_abort_bytes); + + PG_RETURN_INT64(result); +} diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index ac02061..83969a5 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -5393,6 +5393,30 @@ proargmodes => '{i,i,o,o,o,o,o,o,o,o,o}', proargnames => '{subid,relid,subid,relid,command,xid,failure_source,failure_count,last_failure,last_failure_message,stats_reset}', prosrc => 'pg_stat_get_subscription_error' }, +{ oid => '8525', descr => 'statistics: number of transactions commit for a subscription', + proname => 'pg_stat_get_subscription_xact_commit', proisstrict => 'f', provolatile => 's', + proparallel => 'r', prorettype => 'int8', proargtypes => 'oid oid', + prosrc => 'pg_stat_get_subscription_xact_commit' }, +{ oid => '8526', descr => 'statistics: bytes of transactions commit for a subscription', + proname => 'pg_stat_get_subscription_xact_commit_bytes', proisstrict => 'f', provolatile => 's', + proparallel => 'r', prorettype => 'int8', proargtypes => 'oid oid', + prosrc => 'pg_stat_get_subscription_xact_commit_bytes' }, +{ oid => '8527', descr => 'statistics: number of transactions error for a subscription', + proname => 'pg_stat_get_subscription_xact_error', proisstrict => 'f', provolatile => 's', + proparallel => 'r', prorettype => 'int8', proargtypes => 'oid oid', + prosrc => 'pg_stat_get_subscription_xact_error' }, +{ oid => '8528', descr => 'statistics: bytes of transactions error for a subscription', + proname => 'pg_stat_get_subscription_xact_error_bytes', proisstrict => 'f', provolatile => 's', + proparallel => 'r', prorettype => 'int8', proargtypes => 'oid oid', + prosrc => 'pg_stat_get_subscription_xact_error_bytes' }, +{ oid => '8529', descr => 'statistics: number of transactions abort for a subscription', + proname => 'pg_stat_get_subscription_xact_abort', proisstrict => 'f', provolatile => 's', + proparallel => 'r', prorettype => 'int8', proargtypes => 'oid oid', + prosrc => 'pg_stat_get_subscription_xact_abort' }, +{ oid => '8530', descr => 'statistics: bytes of transactions abort for a subscription', + proname => 'pg_stat_get_subscription_xact_abort_bytes', proisstrict => 'f', provolatile => 's', + proparallel => 'r', prorettype => 'int8', proargtypes => 'oid oid', + prosrc => 'pg_stat_get_subscription_xact_abort_bytes' }, { oid => '6118', descr => 'statistics: information about subscription', proname => 'pg_stat_get_subscription', prorows => '10', proisstrict => 'f', proretset => 't', provolatile => 's', proparallel => 'r', diff --git a/src/include/executor/execPartition.h b/src/include/executor/execPartition.h index 694e38b..773e46c 100644 --- a/src/include/executor/execPartition.h +++ b/src/include/executor/execPartition.h @@ -110,6 +110,7 @@ typedef struct PartitionPruneState PartitionPruningData *partprunedata[FLEXIBLE_ARRAY_MEMBER]; } PartitionPruneState; +extern size_t PartitionTupleRoutingSize(void); extern PartitionTupleRouting *ExecSetupPartitionTupleRouting(EState *estate, Relation rel); extern ResultRelInfo *ExecFindPartition(ModifyTableState *mtstate, diff --git a/src/include/pgstat.h b/src/include/pgstat.h index 5ed1319..6109933 100644 --- a/src/include/pgstat.h +++ b/src/include/pgstat.h @@ -67,6 +67,7 @@ typedef enum StatMsgType PGSTAT_MTYPE_RESETSINGLECOUNTER, PGSTAT_MTYPE_RESETSLRUCOUNTER, PGSTAT_MTYPE_RESETREPLSLOTCOUNTER, + PGSTAT_MTYPE_SUBSCRIPTIONSUCCESS, PGSTAT_MTYPE_SUBSCRIPTIONERR, PGSTAT_MTYPE_SUBSCRIPTIONERRPURGE, PGSTAT_MTYPE_SUBSCRIPTIONPURGE, @@ -564,6 +565,7 @@ typedef struct PgStat_MsgSubscriptionErr Oid m_relid; LogicalRepMsgType m_command; TransactionId m_xid; + PgStat_Counter m_bytes; TimestampTz m_failure_time; char m_errmsg[PGSTAT_SUBSCRIPTIONERR_MSGLEN]; } PgStat_MsgSubscriptionErr; @@ -990,13 +992,28 @@ typedef struct PgStat_StatReplSlotEntry * Subscription error statistics kept in the stats collector, representing * an error that occurred during application of logical replicatoin or * initial table synchronization. + * + * Other general stats for transaction on subscription are stored in this entry + * as well, aligned with transaction error stats. Holding the size of computed + * result of bytes during replay depends on the apply error callback not to be + * lost and having general stats in the same place reduces code complexity + * and redundancy. */ typedef struct PgStat_StatSubErrEntry { Oid relid; /* hash table key */ LogicalRepMsgType command; TransactionId xid; - PgStat_Counter failure_count; + + /* transaction stats of subscription */ + PgStat_Counter xact_commit; + PgStat_Counter xact_commit_bytes; + PgStat_Counter xact_error; + PgStat_Counter xact_error_bytes; /* total error counts of this subscription */ + PgStat_Counter xact_abort; + PgStat_Counter xact_abort_bytes; + PgStat_Counter failure_count; /* total error counts of one specific error + continuously happening */ TimestampTz last_failure; char last_errmsg[PGSTAT_SUBSCRIPTIONERR_MSGLEN]; TimestampTz stat_reset_timestamp; @@ -1129,6 +1146,8 @@ extern void pgstat_report_checksum_failure(void); extern void pgstat_report_replslot(const PgStat_StatReplSlotEntry *repSlotStat); extern void pgstat_report_replslot_create(const char *slotname); extern void pgstat_report_replslot_drop(const char *slotname); +extern void pgstat_report_subscription_success(Oid subid, Oid subrel, + LogicalRepMsgType command, PgStat_Counter bytes); extern void pgstat_report_subscription_error(Oid subid, Oid subrelid, Oid relid, LogicalRepMsgType command, TransactionId xid, const char *errmsg); diff --git a/src/include/replication/logicalworker.h b/src/include/replication/logicalworker.h index 2ad61a0..923c8ce 100644 --- a/src/include/replication/logicalworker.h +++ b/src/include/replication/logicalworker.h @@ -15,5 +15,9 @@ extern void ApplyWorkerMain(Datum main_arg); extern bool IsLogicalWorker(void); +extern int64 get_apply_error_context_xact_size(void); +extern void add_apply_error_context_xact_size(int64 size); +extern void reset_apply_error_context_xact_size(void); + #endif /* LOGICALWORKER_H */ diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out index 3719b8a..7a323c6 100644 --- a/src/test/regress/expected/rules.out +++ b/src/test/regress/expected/rules.out @@ -2091,9 +2091,34 @@ pg_stat_subscription| SELECT su.oid AS subid, st.last_msg_send_time, st.last_msg_receipt_time, st.latest_end_lsn, + pg_stat_get_subscription_xact_commit(su.oid, st.relid) AS xact_commit, + pg_stat_get_subscription_xact_commit_bytes(su.oid, st.relid) AS xact_commit_bytes, + pg_stat_get_subscription_xact_error(su.oid, st.relid) AS xact_error, + pg_stat_get_subscription_xact_error_bytes(su.oid, st.relid) AS xact_error_bytes, + pg_stat_get_subscription_xact_abort(su.oid, st.relid) AS xact_abort, + pg_stat_get_subscription_xact_abort_bytes(su.oid, st.relid) AS xact_abort_bytes, st.latest_end_time FROM (pg_subscription su - LEFT JOIN pg_stat_get_subscription(NULL::oid) st(subid, relid, pid, received_lsn, last_msg_send_time, last_msg_receipt_time, latest_end_lsn, latest_end_time) ON ((st.subid = su.oid))); + LEFT JOIN ( SELECT s.subid, + s.relid, + s.pid, + s.received_lsn, + s.last_msg_send_time, + s.last_msg_receipt_time, + s.latest_end_lsn, + s.latest_end_time + FROM pg_stat_get_subscription(NULL::oid) s(subid, relid, pid, received_lsn, last_msg_send_time, last_msg_receipt_time, latest_end_lsn, latest_end_time) + UNION + SELECT r.srsubid, + r.srrelid, + NULL::integer AS pid, + NULL::pg_lsn AS received_lsn, + NULL::timestamp with time zone AS last_msg_send_time, + NULL::timestamp with time zone AS last_msg_receipt_time, + NULL::pg_lsn AS latest_end_lsn, + NULL::timestamp with time zone AS latest_end_time + FROM pg_subscription_rel r + WHERE (r.srsubstate <> 'r'::"char")) st ON ((st.subid = su.oid))); pg_stat_subscription_errors| SELECT d.datname, sr.subid, s.subname, ^ permalink raw reply [nested|flat] 113+ messages in thread
* RE: Failed transaction statistics to measure the logical replication progress @ 2021-09-22 04:39 [email protected] <[email protected]> parent: [email protected] <[email protected]> 0 siblings, 1 reply; 113+ messages in thread From: [email protected] @ 2021-09-22 04:39 UTC (permalink / raw) To: 'Amit Kapila' <[email protected]>; Masahiko Sawada <[email protected]>; +Cc: [email protected] <[email protected]> Hello Just conducted some cosmetic changes and rebased my patch, using v14 patch-set in [1]. [1] - https://www.postgresql.org/message-id/CAD21AoCO_ZYWZEBw7ziiYoX7Zm1P0L9%3Dd7Jj9YsGEGsT9o6wmw%40mail.g... Best Regards, Takamichi Osumi Attachments: [application/octet-stream] extend_subscription_stats_of_transaction_v05.patch (31.5K, ../../TY2PR01MB489042618A073A2FC7BF62C0EDA29@TY2PR01MB4890.jpnprd01.prod.outlook.com/2-extend_subscription_stats_of_transaction_v05.patch) download | inline diff: diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml index b0e4260..7b0838a 100644 --- a/doc/src/sgml/monitoring.sgml +++ b/doc/src/sgml/monitoring.sgml @@ -3046,6 +3046,64 @@ SELECT pid, wait_event_type, wait_event FROM pg_stat_activity WHERE wait_event i <row> <entry role="catalog_table_entry"><para role="column_definition"> + <structfield>xact_commit</structfield> <type>bigint</type> + </para> + <para> + Number of transactions successfully applied in this subscription + </para></entry> + </row> + + <row> + <entry role="catalog_table_entry"><para role="column_definition"> + <structfield>xact_commit_bytes</structfield> <type>bigint</type> + </para> + <para> + Amount of transactions data successfully applied in this subscription + </para></entry> + </row> + + <row> + <entry role="catalog_table_entry"><para role="column_definition"> + <structfield>xact_error</structfield> <type>bigint</type> + </para> + <para> + Number of transactions unsuccessfully applied in this subscription + </para></entry> + </row> + + <row> + <entry role="catalog_table_entry"><para role="column_definition"> + <structfield>xact_error_bytes</structfield> <type>bigint</type> + </para> + <para> + Amount of transactions data unsuccessfully applied in this subscription + </para></entry> + </row> + + <row> + <entry role="catalog_table_entry"><para role="column_definition"> + <structfield>xact_abort</structfield> <type>bigint</type> + </para> + <para> + Number of transactions aborted in this subscription + </para></entry> + </row> + + <row> + <entry role="catalog_table_entry"><para role="column_definition"> + <structfield>xact_abort_bytes</structfield> <type>bigint</type> + </para> + <para> + Amount of transactions data aborted in this subscription. + Increase <literal>logical_decoding_work_mem</literal> on the publisher + to suppress unnecessary consumed network bandwidth or change in memory + of the subscriber, if unexpected amount of rollbacked transactions are + streamed. + </para></entry> + </row> + + <row> + <entry role="catalog_table_entry"><para role="column_definition"> <structfield>latest_end_time</structfield> <type>timestamp with time zone</type> </para> <para> diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index b0cd8d2..3a2c69b 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -909,10 +909,37 @@ CREATE VIEW pg_stat_subscription AS st.last_msg_send_time, st.last_msg_receipt_time, st.latest_end_lsn, + pg_stat_get_subscription_xact_commit(su.oid, st.relid) AS xact_commit, + pg_stat_get_subscription_xact_commit_bytes(su.oid, st.relid) AS xact_commit_bytes, + pg_stat_get_subscription_xact_error(su.oid, st.relid) AS xact_error, + pg_stat_get_subscription_xact_error_bytes(su.oid, st.relid) AS xact_error_bytes, + pg_stat_get_subscription_xact_abort(su.oid, st.relid) AS xact_abort, + pg_stat_get_subscription_xact_abort_bytes(su.oid, st.relid) AS xact_abort_bytes, st.latest_end_time FROM pg_subscription su - LEFT JOIN pg_stat_get_subscription(NULL) st - ON (st.subid = su.oid); + LEFT JOIN + (SELECT + s.subid, + s.relid, + s.pid, + s.received_lsn, + s.last_msg_send_time, + s.last_msg_receipt_time, + s.latest_end_lsn, + s.latest_end_time + FROM pg_stat_get_subscription(NULL) s + UNION -- acquire relid of table sync worker + SELECT + r.srsubid, + r.srrelid, + NULL as pid, + NULL as received_lsn, + NULL as last_msg_send_time, + NULL as last_msg_receipt_time, + NULL as latest_end_lsn, + NULL as latest_end_time + FROM pg_subscription_rel r WHERE r.srsubstate <> 'r') st + ON (st.subid = su.oid); CREATE VIEW pg_stat_ssl AS SELECT diff --git a/src/backend/executor/execPartition.c b/src/backend/executor/execPartition.c index 5c723bc..107a7aa 100644 --- a/src/backend/executor/execPartition.c +++ b/src/backend/executor/execPartition.c @@ -194,6 +194,17 @@ static void find_matching_subplans_recurse(PartitionPruningData *prunedata, /* + * PartitionTupleRoutingSize - exported to calculate total data size + * of logical replication mesage apply, because this is one of the + * ApplyExecutionData struct members. + */ +size_t +PartitionTupleRoutingSize(void) +{ + return sizeof(PartitionTupleRouting); +} + +/* * ExecSetupPartitionTupleRouting - sets up information needed during * tuple routing for partitioned tables, encapsulates it in * PartitionTupleRouting, and returns it. diff --git a/src/backend/postmaster/pgstat.c b/src/backend/postmaster/pgstat.c index d3bc694..6993f83 100644 --- a/src/backend/postmaster/pgstat.c +++ b/src/backend/postmaster/pgstat.c @@ -56,6 +56,7 @@ #include "postmaster/postmaster.h" #include "replication/slot.h" #include "replication/walsender.h" +#include "replication/logicalworker.h" #include "storage/backendid.h" #include "storage/dsm.h" #include "storage/fd.h" @@ -385,6 +386,7 @@ static void pgstat_recv_connect(PgStat_MsgConnect *msg, int len); static void pgstat_recv_disconnect(PgStat_MsgDisconnect *msg, int len); static void pgstat_recv_replslot(PgStat_MsgReplSlot *msg, int len); static void pgstat_recv_tempfile(PgStat_MsgTempFile *msg, int len); +static void pgstat_recv_subscription_success(PgStat_MsgSubscriptionErr *msg, int len); static void pgstat_recv_subscription_error(PgStat_MsgSubscriptionErr *msg, int len); static void pgstat_recv_subscription_error_reset(PgStat_MsgSubscriptionErrReset *msg, int len); @@ -2066,6 +2068,37 @@ pgstat_report_replslot_drop(const char *slotname) } /* ---------- + * pgstat_report_subscription_success() - + * + * Tell the collector about the subscription success. + * ---------- + */ +void +pgstat_report_subscription_success(Oid subid, Oid subrel, + LogicalRepMsgType command, PgStat_Counter bytes) +{ + PgStat_MsgSubscriptionErr msg; + + Assert(command == 0 /* table sync worker */ || + command == LOGICAL_REP_MSG_COMMIT || + command == LOGICAL_REP_MSG_COMMIT_PREPARED || + command == LOGICAL_REP_MSG_STREAM_ABORT); + + pgstat_setheader(&msg.m_hdr, PGSTAT_MTYPE_SUBSCRIPTIONSUCCESS); + msg.m_subid = subid; + msg.m_subrelid = subrel; + msg.m_relid = InvalidOid; + msg.m_command = command; + msg.m_xid = InvalidTransactionId; + msg.m_bytes = bytes; + msg.m_failure_time = GetCurrentTimestamp(); + msg.m_errmsg[0] = '\0'; + pgstat_send(&msg, sizeof(PgStat_MsgSubscriptionErr)); + + reset_apply_error_context_xact_size(); +} + +/* ---------- * pgstat_report_subscription_error() - * * Tell the collector about the subscription error. @@ -2088,10 +2121,13 @@ pgstat_report_subscription_error(Oid subid, Oid subrelid, Oid relid, msg.m_relid = relid; msg.m_command = command; msg.m_xid = xid; + msg.m_bytes = get_apply_error_context_xact_size(); msg.m_failure_time = GetCurrentTimestamp(); strlcpy(msg.m_errmsg, errmsg, PGSTAT_SUBSCRIPTIONERR_MSGLEN); pgstat_send(&msg, len); + + reset_apply_error_context_xact_size(); } /* ---------- @@ -3809,6 +3845,10 @@ PgstatCollectorMain(int argc, char *argv[]) pgstat_recv_disconnect(&msg.msg_disconnect, len); break; + case PGSTAT_MTYPE_SUBSCRIPTIONSUCCESS: + pgstat_recv_subscription_success(&msg.msg_subscriptionerr, len); + break; + case PGSTAT_MTYPE_SUBSCRIPTIONERR: pgstat_recv_subscription_error(&msg.msg_subscriptionerr, len); break; @@ -6271,6 +6311,44 @@ pgstat_recv_funcpurge(PgStat_MsgFuncpurge *msg, int len) } /* ---------- + * pgstat_recv_subscription_success() - + * + * Process a SUBSCRIPTIONERR message. + * ---------- + */ +static void +pgstat_recv_subscription_success(PgStat_MsgSubscriptionErr *msg, int len) +{ + PgStat_StatSubErrEntry *errent; + + errent = pgstat_get_subscription_error_entry(msg->m_subid, + msg->m_subrelid, + true); + + /* msg from table sync worker */ + if (msg->m_command == 0) + errent->xact_commit++; + else + { + switch(msg->m_command) + { + case LOGICAL_REP_MSG_COMMIT: + case LOGICAL_REP_MSG_COMMIT_PREPARED: + errent->xact_commit++; + errent->xact_commit_bytes += msg->m_bytes; + break; + case LOGICAL_REP_MSG_STREAM_ABORT: + errent->xact_abort++; + errent->xact_abort_bytes = msg->m_bytes; + break; + default: + elog(ERROR, "unexpected command type"); + } + } + errent->last_failure = msg->m_failure_time; +} + +/* ---------- * pgstat_recv_subscription_error() - * * Process a SUBSCRIPTIONERR message. @@ -6308,6 +6386,8 @@ pgstat_recv_subscription_error(PgStat_MsgSubscriptionErr *msg, int len) /* update the error entry */ errent->command = msg->m_command; errent->xid = msg->m_xid; + errent->xact_error++; + errent->xact_error_bytes += msg->m_bytes; errent->failure_count++; errent->last_failure = msg->m_failure_time; strlcpy(errent->last_errmsg, msg->m_errmsg, PGSTAT_SUBSCRIPTIONERR_MSGLEN); @@ -6648,6 +6728,12 @@ pgstat_reset_subscription_error_entry(PgStat_StatSubErrEntry *errent, { errent->command = 0; errent->xid = InvalidTransactionId; + errent->xact_commit = 0; + errent->xact_commit_bytes = 0; + errent->xact_error = 0; + errent->xact_error_bytes = 0; + errent->xact_abort = 0; + errent->xact_abort_bytes = 0; errent->failure_count = 0; errent->last_failure = 0; errent->last_errmsg[0] = '\0'; diff --git a/src/backend/replication/logical/proto.c b/src/backend/replication/logical/proto.c index 9f5bf4b..2e1fc94 100644 --- a/src/backend/replication/logical/proto.c +++ b/src/backend/replication/logical/proto.c @@ -17,6 +17,7 @@ #include "catalog/pg_type.h" #include "libpq/pqformat.h" #include "replication/logicalproto.h" +#include "replication/logicalworker.h" #include "utils/lsyscache.h" #include "utils/syscache.h" diff --git a/src/backend/replication/logical/tablesync.c b/src/backend/replication/logical/tablesync.c index f07983a..7ba7486 100644 --- a/src/backend/replication/logical/tablesync.c +++ b/src/backend/replication/logical/tablesync.c @@ -1150,6 +1150,15 @@ copy_table_done: SpinLockRelease(&MyLogicalRepWorker->relmutex); /* + * Update the stats for table sync. We don't record the bytes of + * table synchronization. + */ + pgstat_report_subscription_success(MyLogicalRepWorker->subid, + MyLogicalRepWorker->relid, + 0, /* no corresponding logical message type */ + 0); + + /* * Finally, wait until the main apply worker tells us to catch up and then * return to let LogicalRepApplyLoop do it. */ diff --git a/src/backend/replication/logical/worker.c b/src/backend/replication/logical/worker.c index fea0e81..62610b6 100644 --- a/src/backend/replication/logical/worker.c +++ b/src/backend/replication/logical/worker.c @@ -222,6 +222,20 @@ typedef struct ApplyErrorCallbackArg LogicalRepMsgType command; /* 0 if invalid */ LogicalRepRelMapEntry *rel; + /* + * Store data size of this transaction. + * + * The byte size of transaction on the publisher is calculated + * by ReorderBufferChangeSize() based on the ReorderBufferChange + * structure. But on the subscriber, consumed resources are + * not same as the publisher's decoding process and necessary + * to compute those in different way. Then, the exact same byte + * size is not restored on the subscriber usually. Further, in + * order to give accurate bytes even in the case of error, add + * byte size to this value step by step. + */ + PgStat_Counter bytes; + /* Remote node information */ int remote_attnum; /* -1 if invalid */ TransactionId remote_xid; @@ -232,6 +246,7 @@ static ApplyErrorCallbackArg apply_error_callback_arg = { .command = 0, .rel = NULL, + .bytes = 0, .remote_attnum = -1, .remote_xid = InvalidTransactionId, .ts = 0, @@ -319,6 +334,7 @@ static void maybe_reread_subscription(void); /* prototype needed because of stream_commit */ static void apply_dispatch(StringInfo s); +void update_apply_change_size(LogicalRepMsgType action, void *data); static void apply_handle_commit_internal(LogicalRepCommitData *commit_data); static void apply_handle_insert_internal(ApplyExecutionData *edata, @@ -852,6 +868,10 @@ apply_handle_commit(StringInfo s) /* Process any tables that are being synchronized in parallel. */ process_syncing_tables(commit_data.end_lsn); + pgstat_report_subscription_success(MySubscription->oid, + InvalidOid, + LOGICAL_REP_MSG_COMMIT, + apply_error_callback_arg.bytes); pgstat_report_activity(STATE_IDLE, NULL); reset_apply_error_context_info(); @@ -965,6 +985,7 @@ apply_handle_prepare(StringInfo s) /* Process any tables that are being synchronized in parallel. */ process_syncing_tables(prepare_data.end_lsn); + update_apply_change_size(LOGICAL_REP_MSG_PREPARE, NULL); pgstat_report_activity(STATE_IDLE, NULL); reset_apply_error_context_info(); } @@ -1006,6 +1027,12 @@ apply_handle_commit_prepared(StringInfo s) /* Process any tables that are being synchronized in parallel. */ process_syncing_tables(prepare_data.end_lsn); + update_apply_change_size(LOGICAL_REP_MSG_COMMIT_PREPARED, NULL); + pgstat_report_subscription_success(MySubscription->oid, + InvalidOid, + LOGICAL_REP_MSG_COMMIT_PREPARED, + apply_error_callback_arg.bytes); + pgstat_report_activity(STATE_IDLE, NULL); reset_apply_error_context_info(); } @@ -1193,6 +1220,7 @@ apply_handle_stream_start(StringInfo s) MyLogicalRepWorker->stream_fileset = palloc(sizeof(FileSet)); FileSetInit(MyLogicalRepWorker->stream_fileset); + update_apply_change_size(LOGICAL_REP_MSG_STREAM_START, NULL); MemoryContextSwitchTo(oldctx); } @@ -1345,6 +1373,10 @@ apply_handle_stream_abort(StringInfo s) if (is_skipping_changes()) stop_skipping_changes(InvalidXLogRecPtr, 0); + pgstat_report_subscription_success(MySubscription->oid, + InvalidOid, + LOGICAL_REP_MSG_STREAM_ABORT, + apply_error_callback_arg.bytes); reset_apply_error_context_info(); } @@ -1494,6 +1526,10 @@ apply_handle_stream_commit(StringInfo s) stop_skipping_changes(commit_data.end_lsn, commit_data.committime); store_flush_position(commit_data.end_lsn); + + /* Update the size of change in memory of this transaction */ + update_apply_change_size(LOGICAL_REP_MSG_STREAM_COMMIT, NULL); + in_remote_transaction = false; } else @@ -1646,6 +1682,8 @@ apply_handle_insert(StringInfo s) slot_fill_defaults(rel, estate, remoteslot); MemoryContextSwitchTo(oldctx); + update_apply_change_size(LOGICAL_REP_MSG_INSERT, rel); + /* For a partitioned table, insert the tuple into a partition. */ if (rel->localrel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE) apply_handle_tuple_routing(edata, @@ -1803,6 +1841,8 @@ apply_handle_update(StringInfo s) has_oldtup ? &oldtup : &newtup); MemoryContextSwitchTo(oldctx); + update_apply_change_size(LOGICAL_REP_MSG_UPDATE, rel); + /* For a partitioned table, apply update to correct partition. */ if (rel->localrel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE) apply_handle_tuple_routing(edata, @@ -1937,6 +1977,7 @@ apply_handle_delete(StringInfo s) slot_store_data(remoteslot, rel, &oldtup); MemoryContextSwitchTo(oldctx); + update_apply_change_size(LOGICAL_REP_MSG_DELETE, rel); /* For a partitioned table, apply delete to correct partition. */ if (rel->localrel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE) apply_handle_tuple_routing(edata, @@ -2498,6 +2539,118 @@ apply_dispatch(StringInfo s) apply_error_callback_arg.command = saved_command; } + +/* + * Subscriber side implementation equivalent to ReorderBufferChangeSize + * of the publisher. + * + * According to the logical replication message type, record major + * resource consumptions of this subscription for each message. + * At present, do not collect data from generic functions to keep + * code simplicity. Also, by adding multiple values at once, + * reduce number of function calls. + * + * 'data' controls detail handling of data size calculation. + */ +void +update_apply_change_size(LogicalRepMsgType action, void *data) +{ + int64 size = 0; + LogicalRepRelMapEntry *relmapentry; + LogicalRepRelation *reprelation; + int *stream_write_len; + + /* + * In streaming mode, stream_write_change is called + * instead of immediate apply. List up the messages types + * that can be caught by handle_streamed_transaction and + * treat the write length as the size of transaction so + * that we can export it as part of pg_stat_subscription. + */ + if (in_streamed_transaction && + (action == LOGICAL_REP_MSG_INSERT || + action == LOGICAL_REP_MSG_UPDATE || + action == LOGICAL_REP_MSG_DELETE || + action == LOGICAL_REP_MSG_TRUNCATE || + action == LOGICAL_REP_MSG_RELATION || + action == LOGICAL_REP_MSG_TYPE)) + { + stream_write_len = (int *) data; + size += *stream_write_len; + add_apply_error_context_xact_size(size); + return; + } + + switch (action) + { + /* No special consumption except for generic functions */ + case LOGICAL_REP_MSG_BEGIN: + case LOGICAL_REP_MSG_COMMIT: + case LOGICAL_REP_MSG_TRUNCATE: + case LOGICAL_REP_MSG_TYPE: + case LOGICAL_REP_MSG_ORIGIN: + case LOGICAL_REP_MSG_MESSAGE: + case LOGICAL_REP_MSG_STREAM_STOP: + case LOGICAL_REP_MSG_STREAM_ABORT: + case LOGICAL_REP_MSG_BEGIN_PREPARE: + case LOGICAL_REP_MSG_PREPARE: + case LOGICAL_REP_MSG_ROLLBACK_PREPARED: + break; + + case LOGICAL_REP_MSG_INSERT: + case LOGICAL_REP_MSG_UPDATE: + case LOGICAL_REP_MSG_DELETE: + Assert(data != NULL); + + /* + * Compute size based on ApplyExecutionData. + * The size of LogicalRepRelMapEntry can be skipped because + * it is obtained from hash_search in logicalrep_rel_open. + */ + size += sizeof(ApplyExecutionData) + sizeof(EState) + + sizeof(ResultRelInfo) + sizeof(ResultRelInfo); + + /* + * Add some extra size if the target relation is partitioned. + * PartitionTupleRouting isn't exported. Therefore, call the + * function that returns its size instead. + */ + relmapentry = (LogicalRepRelMapEntry *) data; + if (relmapentry->localrel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE) + size += sizeof(ModifyTableState) + PartitionTupleRoutingSize(); + break; + + + case LOGICAL_REP_MSG_RELATION: + Assert(data != NULL); + + reprelation = (LogicalRepRelation *) data; + /* See logicalrep_read_attrs for the last two */ + size += sizeof(LogicalRepRelation) + + reprelation->natts * sizeof(char *) + + reprelation->natts * sizeof(Oid); + break; + + case LOGICAL_REP_MSG_STREAM_START: + size += sizeof(FileSet); + break; + + case LOGICAL_REP_MSG_STREAM_COMMIT: + case LOGICAL_REP_MSG_COMMIT_PREPARED: + case LOGICAL_REP_MSG_STREAM_PREPARE: + size += sizeof(FlushPosition); + break; + + default: + ereport(ERROR, + (errcode(ERRCODE_PROTOCOL_VIOLATION), + errmsg("invalid logical replication message type \"%c\"", action))); + } + /* update the total size of consumption */ + add_apply_error_context_xact_size(size); +} + + /* * Figure out which write/flush positions to report to the walsender process. * @@ -3352,6 +3505,7 @@ static void stream_write_change(char action, StringInfo s) { int len; + int total_len; Assert(in_streamed_transaction); Assert(TransactionIdIsValid(stream_xid)); @@ -3370,6 +3524,9 @@ stream_write_change(char action, StringInfo s) len = (s->len - s->cursor); BufFileWrite(stream_fd, &s->data[s->cursor], len); + + total_len = (s->len - s->cursor) * 2 + sizeof(char) + sizeof(action); + update_apply_change_size(action, &total_len); } /* @@ -3734,6 +3891,27 @@ set_apply_error_context_xact(TransactionId xid, TimestampTz ts) apply_error_callback_arg.ts = ts; } +/* Exported so that stats collector can utilize this value */ +int64 +get_apply_error_context_xact_size(void) +{ + return apply_error_callback_arg.bytes; +} + +/* Add size to apply error callback bytes */ +void +add_apply_error_context_xact_size(int64 size) +{ + apply_error_callback_arg.bytes += size; +} + +/* Reset bytes information of apply error callback */ +void +reset_apply_error_context_xact_size(void) +{ + apply_error_callback_arg.bytes = 0; +} + /* Reset all information of apply error callback */ static inline void reset_apply_error_context_info(void) diff --git a/src/backend/utils/adt/pgstatfuncs.c b/src/backend/utils/adt/pgstatfuncs.c index 63e7a4b..9630754 100644 --- a/src/backend/utils/adt/pgstatfuncs.c +++ b/src/backend/utils/adt/pgstatfuncs.c @@ -2491,3 +2491,136 @@ pg_stat_get_subscription_error(PG_FUNCTION_ARGS) /* Returns the record as Datum */ PG_RETURN_DATUM(HeapTupleGetDatum(heap_form_tuple(tupdesc, values, nulls))); } + +Datum +pg_stat_get_subscription_xact_commit(PG_FUNCTION_ARGS) +{ + Oid subid = PG_GETARG_OID(0); + Oid relid; + int64 result; + PgStat_StatSubErrEntry *errent; + + if (PG_ARGISNULL(1)) + relid = InvalidOid; + else + relid = PG_GETARG_OID(1); + + /* Get subscription error entry */ + if ((errent = pgstat_fetch_subscription_error(subid, relid)) == NULL) + result = 0; + else + result = (int64) (errent->xact_commit); + + PG_RETURN_INT64(result); +} + +Datum +pg_stat_get_subscription_xact_commit_bytes(PG_FUNCTION_ARGS) +{ + Oid subid = PG_GETARG_OID(0); + Oid relid; + int64 result; + PgStat_StatSubErrEntry *errent; + + if (PG_ARGISNULL(1)) + relid = InvalidOid; + else + relid = PG_GETARG_OID(1); + + /* Get subscription error entry */ + if ((errent = pgstat_fetch_subscription_error(subid, relid)) == NULL) + result = 0; + else + result = (int64) (errent->xact_commit_bytes); + + PG_RETURN_INT64(result); +} + + +Datum +pg_stat_get_subscription_xact_error(PG_FUNCTION_ARGS) +{ + Oid subid = PG_GETARG_OID(0); + Oid relid; + int64 result; + PgStat_StatSubErrEntry *errent; + + if (PG_ARGISNULL(1)) + relid = InvalidOid; + else + relid = PG_GETARG_OID(1); + + /* Get subscription error entry */ + if ((errent = pgstat_fetch_subscription_error(subid, relid)) == NULL) + result = 0; + else + result = (int64) (errent->xact_error); + + PG_RETURN_INT64(result); +} + +Datum +pg_stat_get_subscription_xact_error_bytes(PG_FUNCTION_ARGS) +{ + Oid subid = PG_GETARG_OID(0); + Oid relid; + int64 result; + PgStat_StatSubErrEntry *errent; + + if (PG_ARGISNULL(1)) + relid = InvalidOid; + else + relid = PG_GETARG_OID(1); + + /* Get subscription error entry */ + if ((errent = pgstat_fetch_subscription_error(subid, relid)) == NULL) + result = 0; + else + result = (int64) (errent->xact_error_bytes); + + PG_RETURN_INT64(result); +} + +Datum +pg_stat_get_subscription_xact_abort(PG_FUNCTION_ARGS) +{ + Oid subid = PG_GETARG_OID(0); + Oid relid; + int64 result; + PgStat_StatSubErrEntry *errent; + + if (PG_ARGISNULL(1)) + relid = InvalidOid; + else + relid = PG_GETARG_OID(1); + + /* Get subscription error entry */ + if ((errent = pgstat_fetch_subscription_error(subid, relid)) == NULL) + result = 0; + else + result = (int64) (errent->xact_abort); + + PG_RETURN_INT64(result); +} + +Datum +pg_stat_get_subscription_xact_abort_bytes(PG_FUNCTION_ARGS) +{ + Oid subid = PG_GETARG_OID(0); + Oid relid; + int64 result; + PgStat_StatSubErrEntry *errent; + + if (PG_ARGISNULL(1)) + relid = InvalidOid; + else + relid = PG_GETARG_OID(1); + + /* Get subscription error entry */ + if ((errent = pgstat_fetch_subscription_error(subid, relid)) == NULL) + result = 0; + else + result = (int64) (errent->xact_abort_bytes); + + PG_RETURN_INT64(result); +} diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index ac02061..83969a5 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -5393,6 +5393,30 @@ proargmodes => '{i,i,o,o,o,o,o,o,o,o,o}', proargnames => '{subid,relid,subid,relid,command,xid,failure_source,failure_count,last_failure,last_failure_message,stats_reset}', prosrc => 'pg_stat_get_subscription_error' }, +{ oid => '8525', descr => 'statistics: number of transactions commit for a subscription', + proname => 'pg_stat_get_subscription_xact_commit', proisstrict => 'f', provolatile => 's', + proparallel => 'r', prorettype => 'int8', proargtypes => 'oid oid', + prosrc => 'pg_stat_get_subscription_xact_commit' }, +{ oid => '8526', descr => 'statistics: bytes of transactions commit for a subscription', + proname => 'pg_stat_get_subscription_xact_commit_bytes', proisstrict => 'f', provolatile => 's', + proparallel => 'r', prorettype => 'int8', proargtypes => 'oid oid', + prosrc => 'pg_stat_get_subscription_xact_commit_bytes' }, +{ oid => '8527', descr => 'statistics: number of transactions error for a subscription', + proname => 'pg_stat_get_subscription_xact_error', proisstrict => 'f', provolatile => 's', + proparallel => 'r', prorettype => 'int8', proargtypes => 'oid oid', + prosrc => 'pg_stat_get_subscription_xact_error' }, +{ oid => '8528', descr => 'statistics: bytes of transactions error for a subscription', + proname => 'pg_stat_get_subscription_xact_error_bytes', proisstrict => 'f', provolatile => 's', + proparallel => 'r', prorettype => 'int8', proargtypes => 'oid oid', + prosrc => 'pg_stat_get_subscription_xact_error_bytes' }, +{ oid => '8529', descr => 'statistics: number of transactions abort for a subscription', + proname => 'pg_stat_get_subscription_xact_abort', proisstrict => 'f', provolatile => 's', + proparallel => 'r', prorettype => 'int8', proargtypes => 'oid oid', + prosrc => 'pg_stat_get_subscription_xact_abort' }, +{ oid => '8530', descr => 'statistics: bytes of transactions abort for a subscription', + proname => 'pg_stat_get_subscription_xact_abort_bytes', proisstrict => 'f', provolatile => 's', + proparallel => 'r', prorettype => 'int8', proargtypes => 'oid oid', + prosrc => 'pg_stat_get_subscription_xact_abort_bytes' }, { oid => '6118', descr => 'statistics: information about subscription', proname => 'pg_stat_get_subscription', prorows => '10', proisstrict => 'f', proretset => 't', provolatile => 's', proparallel => 'r', diff --git a/src/include/executor/execPartition.h b/src/include/executor/execPartition.h index 694e38b..773e46c 100644 --- a/src/include/executor/execPartition.h +++ b/src/include/executor/execPartition.h @@ -110,6 +110,7 @@ typedef struct PartitionPruneState PartitionPruningData *partprunedata[FLEXIBLE_ARRAY_MEMBER]; } PartitionPruneState; +extern size_t PartitionTupleRoutingSize(void); extern PartitionTupleRouting *ExecSetupPartitionTupleRouting(EState *estate, Relation rel); extern ResultRelInfo *ExecFindPartition(ModifyTableState *mtstate, diff --git a/src/include/pgstat.h b/src/include/pgstat.h index 3cda0c9..c081e76 100644 --- a/src/include/pgstat.h +++ b/src/include/pgstat.h @@ -67,6 +67,7 @@ typedef enum StatMsgType PGSTAT_MTYPE_RESETSINGLECOUNTER, PGSTAT_MTYPE_RESETSLRUCOUNTER, PGSTAT_MTYPE_RESETREPLSLOTCOUNTER, + PGSTAT_MTYPE_SUBSCRIPTIONSUCCESS, PGSTAT_MTYPE_SUBSCRIPTIONERR, PGSTAT_MTYPE_SUBSCRIPTIONERRRESET, PGSTAT_MTYPE_SUBSCRIPTIONERRPURGE, @@ -564,6 +565,7 @@ typedef struct PgStat_MsgSubscriptionErr Oid m_relid; LogicalRepMsgType m_command; TransactionId m_xid; + PgStat_Counter m_bytes; TimestampTz m_failure_time; char m_errmsg[PGSTAT_SUBSCRIPTIONERR_MSGLEN]; } PgStat_MsgSubscriptionErr; @@ -1027,13 +1029,28 @@ typedef struct PgStat_StatReplSlotEntry * Subscription error statistics kept in the stats collector, representing * an error that occurred during application of logical replication or * initial table synchronization. + * + * Other general stats for transaction on subscription are stored in this entry + * as well, aligned with transaction error stats. Holding the size of computed + * result of bytes during replay depends on the apply error callback not to be + * lost and having general stats in the same place reduces code complexity + * and redundancy. */ typedef struct PgStat_StatSubErrEntry { Oid relid; /* hash table key */ LogicalRepMsgType command; TransactionId xid; - PgStat_Counter failure_count; + + /* transaction stats of subscription */ + PgStat_Counter xact_commit; + PgStat_Counter xact_commit_bytes; + PgStat_Counter xact_error; + PgStat_Counter xact_error_bytes; /* total error counts of this subscription */ + PgStat_Counter xact_abort; + PgStat_Counter xact_abort_bytes; + PgStat_Counter failure_count; /* total error counts of one specific error + continuously happening */ TimestampTz last_failure; char last_errmsg[PGSTAT_SUBSCRIPTIONERR_MSGLEN]; TimestampTz stat_reset_timestamp; @@ -1171,6 +1188,8 @@ extern void pgstat_report_checksum_failure(void); extern void pgstat_report_replslot(const PgStat_StatReplSlotEntry *repSlotStat); extern void pgstat_report_replslot_create(const char *slotname); extern void pgstat_report_replslot_drop(const char *slotname); +extern void pgstat_report_subscription_success(Oid subid, Oid subrel, + LogicalRepMsgType command, PgStat_Counter bytes); extern void pgstat_report_subscription_error(Oid subid, Oid subrelid, Oid relid, LogicalRepMsgType command, TransactionId xid, const char *errmsg); diff --git a/src/include/replication/logicalworker.h b/src/include/replication/logicalworker.h index 2ad61a0..923c8ce 100644 --- a/src/include/replication/logicalworker.h +++ b/src/include/replication/logicalworker.h @@ -15,5 +15,9 @@ extern void ApplyWorkerMain(Datum main_arg); extern bool IsLogicalWorker(void); +extern int64 get_apply_error_context_xact_size(void); +extern void add_apply_error_context_xact_size(int64 size); +extern void reset_apply_error_context_xact_size(void); + #endif /* LOGICALWORKER_H */ diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out index 3719b8a..7a323c6 100644 --- a/src/test/regress/expected/rules.out +++ b/src/test/regress/expected/rules.out @@ -2091,9 +2091,34 @@ pg_stat_subscription| SELECT su.oid AS subid, st.last_msg_send_time, st.last_msg_receipt_time, st.latest_end_lsn, + pg_stat_get_subscription_xact_commit(su.oid, st.relid) AS xact_commit, + pg_stat_get_subscription_xact_commit_bytes(su.oid, st.relid) AS xact_commit_bytes, + pg_stat_get_subscription_xact_error(su.oid, st.relid) AS xact_error, + pg_stat_get_subscription_xact_error_bytes(su.oid, st.relid) AS xact_error_bytes, + pg_stat_get_subscription_xact_abort(su.oid, st.relid) AS xact_abort, + pg_stat_get_subscription_xact_abort_bytes(su.oid, st.relid) AS xact_abort_bytes, st.latest_end_time FROM (pg_subscription su - LEFT JOIN pg_stat_get_subscription(NULL::oid) st(subid, relid, pid, received_lsn, last_msg_send_time, last_msg_receipt_time, latest_end_lsn, latest_end_time) ON ((st.subid = su.oid))); + LEFT JOIN ( SELECT s.subid, + s.relid, + s.pid, + s.received_lsn, + s.last_msg_send_time, + s.last_msg_receipt_time, + s.latest_end_lsn, + s.latest_end_time + FROM pg_stat_get_subscription(NULL::oid) s(subid, relid, pid, received_lsn, last_msg_send_time, last_msg_receipt_time, latest_end_lsn, latest_end_time) + UNION + SELECT r.srsubid, + r.srrelid, + NULL::integer AS pid, + NULL::pg_lsn AS received_lsn, + NULL::timestamp with time zone AS last_msg_send_time, + NULL::timestamp with time zone AS last_msg_receipt_time, + NULL::pg_lsn AS latest_end_lsn, + NULL::timestamp with time zone AS latest_end_time + FROM pg_subscription_rel r + WHERE (r.srsubstate <> 'r'::"char")) st ON ((st.subid = su.oid))); pg_stat_subscription_errors| SELECT d.datname, sr.subid, s.subname, ^ permalink raw reply [nested|flat] 113+ messages in thread
* Re: Failed transaction statistics to measure the logical replication progress @ 2021-09-27 04:42 Amit Kapila <[email protected]> parent: [email protected] <[email protected]> 0 siblings, 1 reply; 113+ messages in thread From: Amit Kapila @ 2021-09-27 04:42 UTC (permalink / raw) To: [email protected] <[email protected]>; +Cc: Masahiko Sawada <[email protected]>; [email protected] <[email protected]> On Wed, Sep 22, 2021 at 10:10 AM [email protected] <[email protected]> wrote: > > Just conducted some cosmetic changes > and rebased my patch, using v14 patch-set in [1]. > IIUC, this proposal will allow new xact stats for subscriptions via pg_stat_subscription. One thing that is not clear to me in this patch is that why you choose a different way to store these stats than the existing stats in that view? AFAICS, the other existing stats are stored in-memory in LogicalRepWorker whereas these new stats are stored/fetched via stats collector means these will persist. Isn't it better to be consistent here? I am not sure which is a more appropriate way to store these stats and I would like to hear your and other's thoughts on that matter but it appears a bit awkward to me that some of the stats in the same view are persistent and others are in-memory. -- With Regards, Amit Kapila. ^ permalink raw reply [nested|flat] 113+ messages in thread
* RE: Failed transaction statistics to measure the logical replication progress @ 2021-09-28 01:55 [email protected] <[email protected]> parent: Amit Kapila <[email protected]> 0 siblings, 1 reply; 113+ messages in thread From: [email protected] @ 2021-09-28 01:55 UTC (permalink / raw) To: 'Amit Kapila' <[email protected]>; +Cc: Masahiko Sawada <[email protected]>; [email protected] <[email protected]> On Monday, September 27, 2021 1:42 PM Amit Kapila <[email protected]> wrote: > On Wed, Sep 22, 2021 at 10:10 AM [email protected] > <[email protected]> wrote: > > > > Just conducted some cosmetic changes > > and rebased my patch, using v14 patch-set in [1]. > > > > IIUC, this proposal will allow new xact stats for subscriptions via > pg_stat_subscription. One thing that is not clear to me in this patch is that why > you choose a different way to store these stats than the existing stats in that > view? AFAICS, the other existing stats are stored in-memory in > LogicalRepWorker whereas these new stats are stored/fetched via stats > collector means these will persist. Isn't it better to be consistent here? I am not > sure which is a more appropriate way to store these stats and I would like to > hear your and other's thoughts on that matter but it appears a bit awkward to > me that some of the stats in the same view are persistent and others are > in-memory. Yeah, existing stats values of pg_stat_subscription are in-memory. I thought xact stats should survive over the restart, to summarize and show all accumulative transaction values on one subscription for user. But, your pointing out is reasonable, mixing two types can be awkward and lack of consistency. Then, if, we proceed in this direction, the place to implement those stats would be on the LogicalRepWorker struct, instead ? On one hand, what confuses me is that in another thread of feature to skip xid, I wondered if Sawada-san has started to take those xact stats into account (probably in his patch-set), because the stats values this thread is taking care of are listed up in the thread. If that's true, its thread and this thread are getting really close. So, IIUC, we have to discuss this point as well. Best Regards, Takamichi Osumi ^ permalink raw reply [nested|flat] 113+ messages in thread
* Re: Failed transaction statistics to measure the logical replication progress @ 2021-09-28 04:54 Amit Kapila <[email protected]> parent: [email protected] <[email protected]> 0 siblings, 1 reply; 113+ messages in thread From: Amit Kapila @ 2021-09-28 04:54 UTC (permalink / raw) To: [email protected] <[email protected]>; +Cc: Masahiko Sawada <[email protected]>; [email protected] <[email protected]> On Tue, Sep 28, 2021 at 7:25 AM [email protected] <[email protected]> wrote: > > On Monday, September 27, 2021 1:42 PM Amit Kapila <[email protected]> wrote: > > On Wed, Sep 22, 2021 at 10:10 AM [email protected] > > <[email protected]> wrote: > > > > > > Just conducted some cosmetic changes > > > and rebased my patch, using v14 patch-set in [1]. > > > > > > > IIUC, this proposal will allow new xact stats for subscriptions via > > pg_stat_subscription. One thing that is not clear to me in this patch is that why > > you choose a different way to store these stats than the existing stats in that > > view? AFAICS, the other existing stats are stored in-memory in > > LogicalRepWorker whereas these new stats are stored/fetched via stats > > collector means these will persist. Isn't it better to be consistent here? I am not > > sure which is a more appropriate way to store these stats and I would like to > > hear your and other's thoughts on that matter but it appears a bit awkward to > > me that some of the stats in the same view are persistent and others are > > in-memory. > Yeah, existing stats values of pg_stat_subscription are in-memory. > I thought xact stats should survive over the restart, > to summarize and show all accumulative transaction values > on one subscription for user. But, your pointing out is reasonable, > mixing two types can be awkward and lack of consistency. > > Then, if, we proceed in this direction, > the place to implement those stats > would be on the LogicalRepWorker struct, instead ? > Or, we can make existing stats persistent and then add these stats on top of it. Sawada-San, do you have any thoughts on this matter? > On one hand, what confuses me is that > in another thread of feature to skip xid, > I wondered if Sawada-san has started to take > those xact stats into account (probably in his patch-set), > because the stats values this thread is taking care of are listed up in the thread. > I don't think the Skip Xid patch is going to take care of these additional stats but Sawada-San can confirm. -- With Regards, Amit Kapila. ^ permalink raw reply [nested|flat] 113+ messages in thread
* Re: Failed transaction statistics to measure the logical replication progress @ 2021-09-28 06:04 Masahiko Sawada <[email protected]> parent: Amit Kapila <[email protected]> 0 siblings, 1 reply; 113+ messages in thread From: Masahiko Sawada @ 2021-09-28 06:04 UTC (permalink / raw) To: Amit Kapila <[email protected]>; +Cc: [email protected] <[email protected]>; [email protected] <[email protected]> On Tue, Sep 28, 2021 at 1:54 PM Amit Kapila <[email protected]> wrote: > > On Tue, Sep 28, 2021 at 7:25 AM [email protected] > <[email protected]> wrote: > > > > On Monday, September 27, 2021 1:42 PM Amit Kapila <[email protected]> wrote: > > > On Wed, Sep 22, 2021 at 10:10 AM [email protected] > > > <[email protected]> wrote: > > > > > > > > Just conducted some cosmetic changes > > > > and rebased my patch, using v14 patch-set in [1]. > > > > > > > > > > IIUC, this proposal will allow new xact stats for subscriptions via > > > pg_stat_subscription. One thing that is not clear to me in this patch is that why > > > you choose a different way to store these stats than the existing stats in that > > > view? AFAICS, the other existing stats are stored in-memory in > > > LogicalRepWorker whereas these new stats are stored/fetched via stats > > > collector means these will persist. Isn't it better to be consistent here? I am not > > > sure which is a more appropriate way to store these stats and I would like to > > > hear your and other's thoughts on that matter but it appears a bit awkward to > > > me that some of the stats in the same view are persistent and others are > > > in-memory. > > Yeah, existing stats values of pg_stat_subscription are in-memory. > > I thought xact stats should survive over the restart, > > to summarize and show all accumulative transaction values > > on one subscription for user. But, your pointing out is reasonable, > > mixing two types can be awkward and lack of consistency. I remembered that we have discussed a similar thing when discussing pg_stat_replication_slots view[1]. The slot statistics such as spill_txn, spill_count, and spill_bytes were originally shown in pg_stat_replication, mixing cumulative counters and dynamic counters. But we concluded to separate these cumulative values from pg_stat_replication view and introduce a new pg_stat_replication_slots view. > > > > Then, if, we proceed in this direction, > > the place to implement those stats > > would be on the LogicalRepWorker struct, instead ? > > > > Or, we can make existing stats persistent and then add these stats on > top of it. Sawada-San, do you have any thoughts on this matter? I think that making existing stats including received_lsn and last_msg_receipt_time persistent by using stats collector could cause massive reporting messages. We can report these messages with a certain interval to reduce the amount of messages but we will end up seeing old stats on the view. Another idea could be to have a separate view, say pg_stat_subscription_xact but I'm not sure it's a better idea. > > > On one hand, what confuses me is that > > in another thread of feature to skip xid, > > I wondered if Sawada-san has started to take > > those xact stats into account (probably in his patch-set), > > because the stats values this thread is taking care of are listed up in the thread. > > > > I don't think the Skip Xid patch is going to take care of these > additional stats but Sawada-San can confirm. Yes, I don't take care of these additional stats discussed on this thread. The reason why I recently mentioned these statistics on the skip_xid thread is that since I thought that this patch will be built on top of the subscription error reporting patch that I'm proposing I discussed the extensibility of my patch. Regards, [1] https://www.postgresql.org/message-id/CABUevEwayASgA2GHAQx%3DVtCp6OQh5PVfem6JDQ97gFHka%3D6n1w%40mail... -- Masahiko Sawada EDB: https://www.enterprisedb.com/ ^ permalink raw reply [nested|flat] 113+ messages in thread
* Re: Failed transaction statistics to measure the logical replication progress @ 2021-09-28 10:04 Amit Kapila <[email protected]> parent: Masahiko Sawada <[email protected]> 0 siblings, 4 replies; 113+ messages in thread From: Amit Kapila @ 2021-09-28 10:04 UTC (permalink / raw) To: Masahiko Sawada <[email protected]>; +Cc: [email protected] <[email protected]>; [email protected] <[email protected]> On Tue, Sep 28, 2021 at 11:35 AM Masahiko Sawada <[email protected]> wrote: > > On Tue, Sep 28, 2021 at 1:54 PM Amit Kapila <[email protected]> wrote: > > > > > > > > Then, if, we proceed in this direction, > > > the place to implement those stats > > > would be on the LogicalRepWorker struct, instead ? > > > > > > > Or, we can make existing stats persistent and then add these stats on > > top of it. Sawada-San, do you have any thoughts on this matter? > > I think that making existing stats including received_lsn and > last_msg_receipt_time persistent by using stats collector could cause > massive reporting messages. We can report these messages with a > certain interval to reduce the amount of messages but we will end up > seeing old stats on the view. > Can't we keep the current and new stats both in-memory and persist on disk? So, the persistent stats data will be used to fill the in-memory counters after restarting of workers, otherwise, we will always refer to in-memory values. > Another idea could be to have a separate > view, say pg_stat_subscription_xact but I'm not sure it's a better > idea. > Yeah, that is another idea but I am afraid that having three different views for subscription stats will be too much. I think it would be better if we can display these additional stats via the existing view pg_stat_subscription or the new view pg_stat_subscription_errors (or whatever name we want to give it). -- With Regards, Amit Kapila. ^ permalink raw reply [nested|flat] 113+ messages in thread
* Re: Failed transaction statistics to measure the logical replication progress @ 2021-09-29 05:59 Greg Nancarrow <[email protected]> parent: Amit Kapila <[email protected]> 3 siblings, 1 reply; 113+ messages in thread From: Greg Nancarrow @ 2021-09-29 05:59 UTC (permalink / raw) To: Amit Kapila <[email protected]>; +Cc: Masahiko Sawada <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]> On Tue, Sep 28, 2021 at 8:04 PM Amit Kapila <[email protected]> wrote: > > > Another idea could be to have a separate > > view, say pg_stat_subscription_xact but I'm not sure it's a better > > idea. > > > > Yeah, that is another idea but I am afraid that having three different > views for subscription stats will be too much. I think it would be > better if we can display these additional stats via the existing view > pg_stat_subscription or the new view pg_stat_subscription_errors (or > whatever name we want to give it). > It seems that we have come full-circle in the discussion of how these new stats could be best added. Other than adding a new "pg_stats_subscription_xact" view (which Amit thought would result in too many views) I don't see any clear solution here, because the new xact-related cumulative stats don't really consistently integrate with the existing in-memory stats in pg_stat_subscription, and IMHO the new stats wouldn't integrate well into the existing error-related stats in the "pg_stat_subscription_errors" view (even if its name was changed, that view in any case maintains lowel-level error details and the way error entries are removed doesn't allow the "success" stats to be maintained for the subscription and doesn't fit well with the added "pg_stat_reset_subscription_error" function either). If we can't really add another view like "pg_stats_subscription_xact", it seems we need to find some way the new stats fit more consistently into the existing "pg_stat_subscription" view. Regards, Greg Nancarrow Fujitsu Australia ^ permalink raw reply [nested|flat] 113+ messages in thread
* RE: Failed transaction statistics to measure the logical replication progress @ 2021-09-29 06:05 [email protected] <[email protected]> parent: Amit Kapila <[email protected]> 3 siblings, 1 reply; 113+ messages in thread From: [email protected] @ 2021-09-29 06:05 UTC (permalink / raw) To: 'Amit Kapila' <[email protected]>; Masahiko Sawada <[email protected]>; +Cc: [email protected] <[email protected]> Hi, Thank you, Amit-san and Sawada-san for the discussion. On Tuesday, September 28, 2021 7:05 PM Amit Kapila <[email protected]> wrote: > > Another idea could be to have a separate view, say > > pg_stat_subscription_xact but I'm not sure it's a better idea. > > > > Yeah, that is another idea but I am afraid that having three different > views for subscription stats will be too much. I think it would be > better if we can display these additional stats via the existing view > pg_stat_subscription or the new view pg_stat_subscription_errors (or > whatever name we want to give it). pg_stat_subscription_errors specializes in showing an error record. So, it would be awkward to combine it with other normal xact stats. > > > > Then, if, we proceed in this direction, the place to implement > > > > those stats would be on the LogicalRepWorker struct, instead ? > > > > > > > > > > Or, we can make existing stats persistent and then add these stats > > > on top of it. Sawada-San, do you have any thoughts on this matter? > > > > I think that making existing stats including received_lsn and > > last_msg_receipt_time persistent by using stats collector could cause > > massive reporting messages. We can report these messages with a > > certain interval to reduce the amount of messages but we will end up > > seeing old stats on the view. > > > > Can't we keep the current and new stats both in-memory and persist on disk? > So, the persistent stats data will be used to fill the in-memory counters after > restarting of workers, otherwise, we will always refer to in-memory values. I felt this isn't impossible. When we have to update the values of the xact stats is the end of message apply for COMMIT, COMMIT PREPARED, STREAM_ABORT and etc or the time when an error happens during apply. Then, if we want, we can update xact stats values at such moments accordingly. I'm thinking that we will have a hash table whose key is a pair of subid + relid and entry is a proposed stats structure and update the entry, depending on the above timings. Here, one thing a bit unclear to me is whether we should move existing stats of pg_stat_subscription (such as last_lsn and reply_lsn) to the hash entry or not. Now, in pg_stat_get_subscription() for pg_stat_subscription view, current stats values are referenced directly from (a copy of) existing LogicalRepCtx->workers. I felt that we need to avoid a situation that some existing data are fetched from LogicalRepWorker and other new xact stats are from the hash in the function, in order to keeping the alignment of the function. Was this correct ? Another thing we need to talk is where we put a new file of contents of pg_stat_subscription. I'm thinking that it's pg_logical/, because above idea does not interact with stats collector any more. Let me know if I miss something. Best Regards, Takamichi Osumi ^ permalink raw reply [nested|flat] 113+ messages in thread
* RE: Failed transaction statistics to measure the logical replication progress @ 2021-09-29 06:27 [email protected] <[email protected]> parent: Greg Nancarrow <[email protected]> 0 siblings, 0 replies; 113+ messages in thread From: [email protected] @ 2021-09-29 06:27 UTC (permalink / raw) To: 'Greg Nancarrow' <[email protected]>; Amit Kapila <[email protected]>; +Cc: Masahiko Sawada <[email protected]>; [email protected] <[email protected]> On Wednesday, September 29, 2021 2:59 PM Greg Nancarrow <[email protected]> wrote: > On Tue, Sep 28, 2021 at 8:04 PM Amit Kapila <[email protected]> > wrote: > > > > > Another idea could be to have a separate view, say > > > pg_stat_subscription_xact but I'm not sure it's a better idea. > > > > > > > Yeah, that is another idea but I am afraid that having three different > > views for subscription stats will be too much. I think it would be > > better if we can display these additional stats via the existing view > > pg_stat_subscription or the new view pg_stat_subscription_errors (or > > whatever name we want to give it). > > It seems that we have come full-circle in the discussion of how these new stats > could be best added. > Other than adding a new "pg_stats_subscription_xact" view (which Amit > thought would result in too many views) I don't see any clear solution here, > because the new xact-related cumulative stats don't really consistently > integrate with the existing in-memory stats in pg_stat_subscription, and IMHO > the new stats wouldn't integrate well into the existing error-related stats in the > "pg_stat_subscription_errors" view (even if its name was changed, that view in > any case maintains lowel-level error details and the way error entries are > removed doesn't allow the "success" stats to be maintained for the > subscription and doesn't fit well with the added > "pg_stat_reset_subscription_error" function either). > If we can't really add another view like "pg_stats_subscription_xact", it seems > we need to find some way the new stats fit more consistently into the existing > "pg_stat_subscription" view. Yeah, I agree with your conclusion. I feel we cannot avoid changing some base of pg_stat_subscription for the xact stats. Best Regards, Takamichi Osumi ^ permalink raw reply [nested|flat] 113+ messages in thread
* Re: Failed transaction statistics to measure the logical replication progress @ 2021-09-29 10:51 Amit Kapila <[email protected]> parent: [email protected] <[email protected]> 0 siblings, 1 reply; 113+ messages in thread From: Amit Kapila @ 2021-09-29 10:51 UTC (permalink / raw) To: [email protected] <[email protected]>; +Cc: Masahiko Sawada <[email protected]>; [email protected] <[email protected]> On Wed, Sep 29, 2021 at 11:35 AM [email protected] <[email protected]> wrote: > > Hi, > > > Thank you, Amit-san and Sawada-san for the discussion. > On Tuesday, September 28, 2021 7:05 PM Amit Kapila <[email protected]> wrote: > > > Another idea could be to have a separate view, say > > > pg_stat_subscription_xact but I'm not sure it's a better idea. > > > > > > > Yeah, that is another idea but I am afraid that having three different > > views for subscription stats will be too much. I think it would be > > better if we can display these additional stats via the existing view > > pg_stat_subscription or the new view pg_stat_subscription_errors (or > > whatever name we want to give it). > pg_stat_subscription_errors specializes in showing an error record. > So, it would be awkward to combine it with other normal xact stats. > > > > > > > Then, if, we proceed in this direction, the place to implement > > > > > those stats would be on the LogicalRepWorker struct, instead ? > > > > > > > > > > > > > Or, we can make existing stats persistent and then add these stats > > > > on top of it. Sawada-San, do you have any thoughts on this matter? > > > > > > I think that making existing stats including received_lsn and > > > last_msg_receipt_time persistent by using stats collector could cause > > > massive reporting messages. We can report these messages with a > > > certain interval to reduce the amount of messages but we will end up > > > seeing old stats on the view. > > > > > > > Can't we keep the current and new stats both in-memory and persist on disk? > > So, the persistent stats data will be used to fill the in-memory counters after > > restarting of workers, otherwise, we will always refer to in-memory values. > I felt this isn't impossible. > When we have to update the values of the xact stats is > the end of message apply for COMMIT, COMMIT PREPARED, STREAM_ABORT and etc > or the time when an error happens during apply. Then, if we want, > we can update xact stats values at such moments accordingly. > I'm thinking that we will have a hash table whose key is a pair of subid + relid > and entry is a proposed stats structure and update the entry, > depending on the above timings. > Are you thinking of a separate hash table then what we are going to create for Sawada-San's patch related to error stats? Isn't it possible to have stats in the same hash table and same file? > Here, one thing a bit unclear to me is > whether we should move existing stats of pg_stat_subscription > (such as last_lsn and reply_lsn) to the hash entry or not. > I think we should move it to hash entry. I think that is an improvement over what we have now because now after restart those stats gets lost. -- With Regards, Amit Kapila. ^ permalink raw reply [nested|flat] 113+ messages in thread
* Re: Failed transaction statistics to measure the logical replication progress @ 2021-09-29 11:18 Masahiko Sawada <[email protected]> parent: Amit Kapila <[email protected]> 3 siblings, 1 reply; 113+ messages in thread From: Masahiko Sawada @ 2021-09-29 11:18 UTC (permalink / raw) To: Amit Kapila <[email protected]>; +Cc: [email protected] <[email protected]>; [email protected] <[email protected]> On Tue, Sep 28, 2021 at 7:04 PM Amit Kapila <[email protected]> wrote: > > On Tue, Sep 28, 2021 at 11:35 AM Masahiko Sawada <[email protected]> wrote: > > > > On Tue, Sep 28, 2021 at 1:54 PM Amit Kapila <[email protected]> wrote: > > > > > > > > > > > Then, if, we proceed in this direction, > > > > the place to implement those stats > > > > would be on the LogicalRepWorker struct, instead ? > > > > > > > > > > Or, we can make existing stats persistent and then add these stats on > > > top of it. Sawada-San, do you have any thoughts on this matter? > > > > I think that making existing stats including received_lsn and > > last_msg_receipt_time persistent by using stats collector could cause > > massive reporting messages. We can report these messages with a > > certain interval to reduce the amount of messages but we will end up > > seeing old stats on the view. > > > > Can't we keep the current and new stats both in-memory and persist on > disk? So, the persistent stats data will be used to fill the in-memory > counters after restarting of workers, otherwise, we will always refer > to in-memory values. Interesting. Probably we can have apply workers and table sync workers send their statistics to the stats collector at exit (before the stats collector shutting down)? And the startup process will restore them at restart? > > > Another idea could be to have a separate > > view, say pg_stat_subscription_xact but I'm not sure it's a better > > idea. > > > > Yeah, that is another idea but I am afraid that having three different > views for subscription stats will be too much. Yeah, I have the same feeling. Regards, -- Masahiko Sawada EDB: https://www.enterprisedb.com/ ^ permalink raw reply [nested|flat] 113+ messages in thread
* Re: Failed transaction statistics to measure the logical replication progress @ 2021-09-29 11:55 Amit Kapila <[email protected]> parent: Masahiko Sawada <[email protected]> 0 siblings, 1 reply; 113+ messages in thread From: Amit Kapila @ 2021-09-29 11:55 UTC (permalink / raw) To: Masahiko Sawada <[email protected]>; +Cc: [email protected] <[email protected]>; [email protected] <[email protected]> On Wed, Sep 29, 2021 at 4:49 PM Masahiko Sawada <[email protected]> wrote: > > On Tue, Sep 28, 2021 at 7:04 PM Amit Kapila <[email protected]> wrote: > > > > On Tue, Sep 28, 2021 at 11:35 AM Masahiko Sawada <[email protected]> wrote: > > > > > > On Tue, Sep 28, 2021 at 1:54 PM Amit Kapila <[email protected]> wrote: > > > > > > > > > > > > > > Then, if, we proceed in this direction, > > > > > the place to implement those stats > > > > > would be on the LogicalRepWorker struct, instead ? > > > > > > > > > > > > > Or, we can make existing stats persistent and then add these stats on > > > > top of it. Sawada-San, do you have any thoughts on this matter? > > > > > > I think that making existing stats including received_lsn and > > > last_msg_receipt_time persistent by using stats collector could cause > > > massive reporting messages. We can report these messages with a > > > certain interval to reduce the amount of messages but we will end up > > > seeing old stats on the view. > > > > > > > Can't we keep the current and new stats both in-memory and persist on > > disk? So, the persistent stats data will be used to fill the in-memory > > counters after restarting of workers, otherwise, we will always refer > > to in-memory values. > > Interesting. Probably we can have apply workers and table sync workers > send their statistics to the stats collector at exit (before the stats > collector shutting down)? And the startup process will restore them at > restart? > I think we do need to send at the exit but we should probably send at some other regular interval as well to avoid losing all the stats after the crash-restart situation. -- With Regards, Amit Kapila. ^ permalink raw reply [nested|flat] 113+ messages in thread
* RE: Failed transaction statistics to measure the logical replication progress @ 2021-09-30 02:23 Osumi, Takamichi/大墨 昂道 <[email protected]> parent: Amit Kapila <[email protected]> 0 siblings, 0 replies; 113+ messages in thread From: Osumi, Takamichi/大墨 昂道 @ 2021-09-30 02:23 UTC (permalink / raw) To: 'Amit Kapila' <[email protected]>; +Cc: Masahiko Sawada <[email protected]>; [email protected] <[email protected]> On Wednesday, September 29, 2021 7:51 PM Amit Kapila <[email protected]> wrote: > On Wed, Sep 29, 2021 at 11:35 AM [email protected] > <[email protected]> wrote: > > Thank you, Amit-san and Sawada-san for the discussion. > > On Tuesday, September 28, 2021 7:05 PM Amit Kapila > <[email protected]> wrote: > > > > Another idea could be to have a separate view, say > > > > pg_stat_subscription_xact but I'm not sure it's a better idea. > > > > > > > > > > Yeah, that is another idea but I am afraid that having three > > > different views for subscription stats will be too much. I think it > > > would be better if we can display these additional stats via the > > > existing view pg_stat_subscription or the new view > > > pg_stat_subscription_errors (or whatever name we want to give it). > > pg_stat_subscription_errors specializes in showing an error record. > > So, it would be awkward to combine it with other normal xact stats. > > > > > > > > > > Then, if, we proceed in this direction, the place to implement > > > > > > those stats would be on the LogicalRepWorker struct, instead ? > > > > > > > > > > > > > > > > Or, we can make existing stats persistent and then add these > > > > > stats on top of it. Sawada-San, do you have any thoughts on this > matter? > > > > > > > > I think that making existing stats including received_lsn and > > > > last_msg_receipt_time persistent by using stats collector could > > > > cause massive reporting messages. We can report these messages > > > > with a certain interval to reduce the amount of messages but we > > > > will end up seeing old stats on the view. > > > > > > > > > > Can't we keep the current and new stats both in-memory and persist on > disk? > > > So, the persistent stats data will be used to fill the in-memory > > > counters after restarting of workers, otherwise, we will always refer to > in-memory values. > > I felt this isn't impossible. > > When we have to update the values of the xact stats is the end of > > message apply for COMMIT, COMMIT PREPARED, STREAM_ABORT and etc > or the > > time when an error happens during apply. Then, if we want, we can > > update xact stats values at such moments accordingly. > > I'm thinking that we will have a hash table whose key is a pair of > > subid + relid and entry is a proposed stats structure and update the > > entry, depending on the above timings. > > > > Are you thinking of a separate hash table then what we are going to create for > Sawada-San's patch related to error stats? Isn't it possible to have stats in the > same hash table and same file? IIUC, this would be possible. At the beginning, I thought we don't use stats collector at all for the xact stats with the existing stats like received_lsn and last_msg_receipt_time, considering the concern of too many reporting messages for them to keep them always updated. But, when we send messages to stats collector only strictly limited times, like successful exit and some regular interval as you explained in your other email, this would disappear and I thought those new xact stats + moved stats can coexist in the hash proposed by you in [1] and we can write those new stats in the same file. Does everyone agree ? > > Here, one thing a bit unclear to me is whether we should move existing > > stats of pg_stat_subscription (such as last_lsn and reply_lsn) to the > > hash entry or not. > > > > I think we should move it to hash entry. I think that is an improvement over what > we have now because now after restart those stats gets lost. Okay ! [1] - https://www.postgresql.org/message-id/CAA4eK1JRCQ-bYnbkwUrvcVcbLURjtiW%2BirFVvXzeG%2Bj%3Dy6jVgA%40ma... Best Regards, Takamichi Osumi ^ permalink raw reply [nested|flat] 113+ messages in thread
* Re: Failed transaction statistics to measure the logical replication progress @ 2021-09-30 02:47 Masahiko Sawada <[email protected]> parent: Amit Kapila <[email protected]> 0 siblings, 1 reply; 113+ messages in thread From: Masahiko Sawada @ 2021-09-30 02:47 UTC (permalink / raw) To: Amit Kapila <[email protected]>; +Cc: [email protected] <[email protected]>; [email protected] <[email protected]> On Wed, Sep 29, 2021 at 8:55 PM Amit Kapila <[email protected]> wrote: > > On Wed, Sep 29, 2021 at 4:49 PM Masahiko Sawada <[email protected]> wrote: > > > > On Tue, Sep 28, 2021 at 7:04 PM Amit Kapila <[email protected]> wrote: > > > > > > On Tue, Sep 28, 2021 at 11:35 AM Masahiko Sawada <[email protected]> wrote: > > > > > > > > On Tue, Sep 28, 2021 at 1:54 PM Amit Kapila <[email protected]> wrote: > > > > > > > > > > > > > > > > > Then, if, we proceed in this direction, > > > > > > the place to implement those stats > > > > > > would be on the LogicalRepWorker struct, instead ? > > > > > > > > > > > > > > > > Or, we can make existing stats persistent and then add these stats on > > > > > top of it. Sawada-San, do you have any thoughts on this matter? > > > > > > > > I think that making existing stats including received_lsn and > > > > last_msg_receipt_time persistent by using stats collector could cause > > > > massive reporting messages. We can report these messages with a > > > > certain interval to reduce the amount of messages but we will end up > > > > seeing old stats on the view. > > > > > > > > > > Can't we keep the current and new stats both in-memory and persist on > > > disk? So, the persistent stats data will be used to fill the in-memory > > > counters after restarting of workers, otherwise, we will always refer > > > to in-memory values. > > > > Interesting. Probably we can have apply workers and table sync workers > > send their statistics to the stats collector at exit (before the stats > > collector shutting down)? And the startup process will restore them at > > restart? > > > > I think we do need to send at the exit but we should probably send at > some other regular interval as well to avoid losing all the stats > after the crash-restart situation. But we clear all statistics collected by the stats collector during crash recovery. No? Regards, -- Masahiko Sawada EDB: https://www.enterprisedb.com/ ^ permalink raw reply [nested|flat] 113+ messages in thread
* RE: Failed transaction statistics to measure the logical replication progress @ 2021-09-30 02:52 Hou, Zhijie/侯 志杰 <[email protected]> parent: Amit Kapila <[email protected]> 3 siblings, 2 replies; 113+ messages in thread From: Hou, Zhijie/侯 志杰 @ 2021-09-30 02:52 UTC (permalink / raw) To: Amit Kapila <[email protected]>; +Cc: Osumi, Takamichi/大墨 昂道 <[email protected]>; [email protected] <[email protected]>; Masahiko Sawada <[email protected]> On Tues, Sep 28, 2021 6:05 PM Amit Kapila <[email protected]> wrote: > On Tue, Sep 28, 2021 at 11:35 AM Masahiko Sawada <[email protected]> wrote: > > > > On Tue, Sep 28, 2021 at 1:54 PM Amit Kapila <[email protected]> > wrote: > > > > > > > > > > > Then, if, we proceed in this direction, > > > > the place to implement those stats > > > > would be on the LogicalRepWorker struct, instead ? > > > > > > > > > > Or, we can make existing stats persistent and then add these stats on > > > top of it. Sawada-San, do you have any thoughts on this matter? > > > > I think that making existing stats including received_lsn and > > last_msg_receipt_time persistent by using stats collector could cause > > massive reporting messages. We can report these messages with a > > certain interval to reduce the amount of messages but we will end up > > seeing old stats on the view. > > > > Can't we keep the current and new stats both in-memory and persist on > disk? So, the persistent stats data will be used to fill the in-memory > counters after restarting of workers, otherwise, we will always refer > to in-memory values. I think this approach works, but I have another concern about it. The current pg_stat_subscription view is listed as "Dynamic Statistics Views" in the document, the data in it seems about the worker process, and the view data shows what the current worker did. But if we keep the new xact stat persist, then it's not what the current worker did, it looks more related to the subscription historic data. Adding a new view seems resonalble, but it will bring another subscription related view which might be too much. OTOH, I can see there are already some different views[1] including xact stat, maybe adding another one is accepatble ? [1] pg_stat_xact_all_tables pg_stat_xact_sys_tables pg_stat_xact_user_tables pg_stat_xact_user_functions Best regards, Hou zj ^ permalink raw reply [nested|flat] 113+ messages in thread
* Re: Failed transaction statistics to measure the logical replication progress @ 2021-09-30 03:12 Amit Kapila <[email protected]> parent: Masahiko Sawada <[email protected]> 0 siblings, 0 replies; 113+ messages in thread From: Amit Kapila @ 2021-09-30 03:12 UTC (permalink / raw) To: Masahiko Sawada <[email protected]>; +Cc: [email protected] <[email protected]>; [email protected] <[email protected]> On Thu, Sep 30, 2021 at 8:18 AM Masahiko Sawada <[email protected]> wrote: > > On Wed, Sep 29, 2021 at 8:55 PM Amit Kapila <[email protected]> wrote: > > > > On Wed, Sep 29, 2021 at 4:49 PM Masahiko Sawada <[email protected]> wrote: > > > > > > On Tue, Sep 28, 2021 at 7:04 PM Amit Kapila <[email protected]> wrote: > > > > > > > > On Tue, Sep 28, 2021 at 11:35 AM Masahiko Sawada <[email protected]> wrote: > > > > > > > > > > On Tue, Sep 28, 2021 at 1:54 PM Amit Kapila <[email protected]> wrote: > > > > > > > > > > > > > > > > > > > > Then, if, we proceed in this direction, > > > > > > > the place to implement those stats > > > > > > > would be on the LogicalRepWorker struct, instead ? > > > > > > > > > > > > > > > > > > > Or, we can make existing stats persistent and then add these stats on > > > > > > top of it. Sawada-San, do you have any thoughts on this matter? > > > > > > > > > > I think that making existing stats including received_lsn and > > > > > last_msg_receipt_time persistent by using stats collector could cause > > > > > massive reporting messages. We can report these messages with a > > > > > certain interval to reduce the amount of messages but we will end up > > > > > seeing old stats on the view. > > > > > > > > > > > > > Can't we keep the current and new stats both in-memory and persist on > > > > disk? So, the persistent stats data will be used to fill the in-memory > > > > counters after restarting of workers, otherwise, we will always refer > > > > to in-memory values. > > > > > > Interesting. Probably we can have apply workers and table sync workers > > > send their statistics to the stats collector at exit (before the stats > > > collector shutting down)? And the startup process will restore them at > > > restart? > > > > > > > I think we do need to send at the exit but we should probably send at > > some other regular interval as well to avoid losing all the stats > > after the crash-restart situation. > > But we clear all statistics collected by the stats collector during > crash recovery. > Right. So, maybe if we do what you are suggesting is sufficient. -- With Regards, Amit Kapila. ^ permalink raw reply [nested|flat] 113+ messages in thread
* RE: Failed transaction statistics to measure the logical replication progress @ 2021-09-30 04:06 Osumi, Takamichi/大墨 昂道 <[email protected]> parent: Hou, Zhijie/侯 志杰 <[email protected]> 1 sibling, 0 replies; 113+ messages in thread From: Osumi, Takamichi/大墨 昂道 @ 2021-09-30 04:06 UTC (permalink / raw) To: Hou, Zhijie/侯 志杰 <[email protected]>; Amit Kapila <[email protected]>; +Cc: [email protected] <[email protected]>; Masahiko Sawada <[email protected]> On Thursday, September 30, 2021 11:53 AM Hou, Zhijie/侯 志杰 <[email protected]> wrote: > On Tues, Sep 28, 2021 6:05 PM Amit Kapila <[email protected]> wrote: > > On Tue, Sep 28, 2021 at 11:35 AM Masahiko Sawada > <[email protected]> wrote: > > > > > > On Tue, Sep 28, 2021 at 1:54 PM Amit Kapila > > > <[email protected]> > > wrote: > > > > > > > > > > > > > > Then, if, we proceed in this direction, the place to implement > > > > > those stats would be on the LogicalRepWorker struct, instead ? > > > > > > > > > > > > > Or, we can make existing stats persistent and then add these stats > > > > on top of it. Sawada-San, do you have any thoughts on this matter? > > > > > > I think that making existing stats including received_lsn and > > > last_msg_receipt_time persistent by using stats collector could > > > cause massive reporting messages. We can report these messages with > > > a certain interval to reduce the amount of messages but we will end > > > up seeing old stats on the view. > > > > > > > Can't we keep the current and new stats both in-memory and persist on > > disk? So, the persistent stats data will be used to fill the in-memory > > counters after restarting of workers, otherwise, we will always refer > > to in-memory values. > > I think this approach works, but I have another concern about it. > > The current pg_stat_subscription view is listed as "Dynamic Statistics Views" > in the document, the data in it seems about the worker process, and the view > data shows what the current worker did. But if we keep the new xact stat > persist, then it's not what the current worker did, it looks more related to the > subscription historic data. > > Adding a new view seems resonalble, but it will bring another subscription > related view which might be too much. OTOH, I can see there are already some > different views[1] including xact stat, maybe adding another one is > accepatble ? I think we'll try to suppress the increment of the numbers of subscription related stats, if the possibility is not denied. In terms of the document you mentioned, I feel I'd need some modifications to it in the patch, based on the change. Best Regards, Takamichi Osumi ^ permalink raw reply [nested|flat] 113+ messages in thread
* Re: Failed transaction statistics to measure the logical replication progress @ 2021-09-30 04:14 Amit Kapila <[email protected]> parent: Hou, Zhijie/侯 志杰 <[email protected]> 1 sibling, 2 replies; 113+ messages in thread From: Amit Kapila @ 2021-09-30 04:14 UTC (permalink / raw) To: Hou, Zhijie/侯 志杰 <[email protected]>; +Cc: Osumi, Takamichi/大墨 昂道 <[email protected]>; [email protected] <[email protected]>; Masahiko Sawada <[email protected]> On Thu, Sep 30, 2021 at 8:22 AM Hou, Zhijie/侯 志杰 <[email protected]> wrote: > > On Tues, Sep 28, 2021 6:05 PM Amit Kapila <[email protected]> wrote: > > > > Can't we keep the current and new stats both in-memory and persist on > > disk? So, the persistent stats data will be used to fill the in-memory > > counters after restarting of workers, otherwise, we will always refer > > to in-memory values. > > I think this approach works, but I have another concern about it. > > The current pg_stat_subscription view is listed as "Dynamic Statistics Views" in > the document, the data in it seems about the worker process, and the view data > shows what the current worker did. But if we keep the new xact stat persist, > then it's not what the current worker did, it looks more related to the > subscription historic data. > I see your point. > Adding a new view seems resonalble, but it will bring another subscription > related view which might be too much. OTOH, I can see there are already some > different views[1] including xact stat, maybe adding another one is accepatble ? > These all views are related to untransmitted to the collector but what we really need is a view similar to pg_stat_archiver or pg_stat_bgwriter which gives information about background workers. Now, the problem as I see is if we go that route then pg_stat_subscription will no longer remain dynamic view and one might consider that as a compatibility break. The other idea I have shared is that we display these stats under the new view introduced by Sawada-San's patch [1] and probably rename that view as pg_stat_subscription_worker where all the stats (xact info and last failure information) about each worker will be displayed. Do you have any opinion on that idea or do you see any problem with it? Sure, we can introduce a new view but I want to avoid it if possible. [1] - https://www.postgresql.org/message-id/CAD21AoDeScrsHhLyEPYqN3sydg6PxAPVBboK%3D30xJfUVihNZDA%40mail.g... -- With Regards, Amit Kapila. ^ permalink raw reply [nested|flat] 113+ messages in thread
* RE: Failed transaction statistics to measure the logical replication progress @ 2021-09-30 07:32 Osumi, Takamichi/大墨 昂道 <[email protected]> parent: Amit Kapila <[email protected]> 1 sibling, 1 reply; 113+ messages in thread From: Osumi, Takamichi/大墨 昂道 @ 2021-09-30 07:32 UTC (permalink / raw) To: 'Amit Kapila' <[email protected]>; Hou, Zhijie/侯 志杰 <[email protected]>; +Cc: [email protected] <[email protected]>; Masahiko Sawada <[email protected]> On Thursday, September 30, 2021 1:15 PM Amit Kapila <[email protected]> wrote: > On Thu, Sep 30, 2021 at 8:22 AM Hou, Zhijie/侯 志杰 > <[email protected]> wrote: > > > > On Tues, Sep 28, 2021 6:05 PM Amit Kapila <[email protected]> > wrote: > > Adding a new view seems resonalble, but it will bring another > > subscription related view which might be too much. OTOH, I can see > > there are already some different views[1] including xact stat, maybe adding > another one is accepatble ? > These all views are related to untransmitted to the collector but what we really > need is a view similar to pg_stat_archiver or pg_stat_bgwriter which gives > information about background workers. > Now, the problem as I see is if we go that route then pg_stat_subscription will > no longer remain dynamic view and one might consider that as a compatibility > break. The other idea I have shared is that we display these stats under the new > view introduced by Sawada-San's patch [1] and probably rename that view as > pg_stat_subscription_worker where all the stats (xact info and last failure > information) about each worker will be displayed. Sorry, all the stats in pg_stat_subscription_worker view ? There was a discussion that the xact info should be displayed from pg_stat_subscription with existing stats in the same (which will be changed to persist), but when your above proposal comes true, the list of pg_stat_subscription_worker's columns will be something like below (when I list up major columns). - subid, subrelid and some other relation attributes required - 5 stats values moved from pg_stat_subscription received_lsn, last_msg_send_time, last_msg_receipt_time, latest_end_lsn, latest_end_time - xact stats xact_commit, xact_commit_bytes, xact_error, xact_error_bytes, xact_abort, xact_abort_bytes, - error stats datname, command, xid, failure_source, failure_count, last_failure, last_failure_message, etc If this is what you imagined, what we can do from left values of pg_stat_subscription would be only finding subscription worker's process id. Is it OK and is this view's alignment acceptable ? Best Regards, Takamichi Osumi ^ permalink raw reply [nested|flat] 113+ messages in thread
* Re: Failed transaction statistics to measure the logical replication progress @ 2021-09-30 11:13 Amit Kapila <[email protected]> parent: Osumi, Takamichi/大墨 昂道 <[email protected]> 0 siblings, 1 reply; 113+ messages in thread From: Amit Kapila @ 2021-09-30 11:13 UTC (permalink / raw) To: Osumi, Takamichi/大墨 昂道 <[email protected]>; +Cc: Hou, Zhijie/侯 志杰 <[email protected]>; [email protected] <[email protected]>; Masahiko Sawada <[email protected]> On Thu, Sep 30, 2021 at 1:02 PM Osumi, Takamichi/大墨 昂道 <[email protected]> wrote: > > On Thursday, September 30, 2021 1:15 PM Amit Kapila <[email protected]> wrote: > > On Thu, Sep 30, 2021 at 8:22 AM Hou, Zhijie/侯 志杰 > > <[email protected]> wrote: > > > > > > On Tues, Sep 28, 2021 6:05 PM Amit Kapila <[email protected]> > > wrote: > > > Adding a new view seems resonalble, but it will bring another > > > subscription related view which might be too much. OTOH, I can see > > > there are already some different views[1] including xact stat, maybe adding > > another one is accepatble ? > > These all views are related to untransmitted to the collector but what we really > > need is a view similar to pg_stat_archiver or pg_stat_bgwriter which gives > > information about background workers. > > Now, the problem as I see is if we go that route then pg_stat_subscription will > > no longer remain dynamic view and one might consider that as a compatibility > > break. The other idea I have shared is that we display these stats under the new > > view introduced by Sawada-San's patch [1] and probably rename that view as > > pg_stat_subscription_worker where all the stats (xact info and last failure > > information) about each worker will be displayed. > Sorry, all the stats in pg_stat_subscription_worker view ? > > There was a discussion that > the xact info should be displayed from pg_stat_subscription > with existing stats in the same (which will be changed to persist), > but when your above proposal comes true, > the list of pg_stat_subscription_worker's columns > will be something like below (when I list up major columns). > > - subid, subrelid and some other relation attributes required > - 5 stats values moved from pg_stat_subscription > received_lsn, last_msg_send_time, last_msg_receipt_time, > latest_end_lsn, latest_end_time > If we go with the view as pg_stat_subscription_worker, we don't need to move the existing stats of pg_stat_subscription into it. There is a clear difference between them, all the stats displayed via pg_stat_subscription are dynamic and won't persist whereas all the stats corresponding to pg_stat_subscription_worker view will persist. -- With Regards, Amit Kapila. ^ permalink raw reply [nested|flat] 113+ messages in thread
* RE: Failed transaction statistics to measure the logical replication progress @ 2021-10-01 14:52 [email protected] <[email protected]> parent: Amit Kapila <[email protected]> 0 siblings, 0 replies; 113+ messages in thread From: [email protected] @ 2021-10-01 14:52 UTC (permalink / raw) To: 'Amit Kapila' <[email protected]>; +Cc: [email protected] <[email protected]>; [email protected] <[email protected]>; Masahiko Sawada <[email protected]> On Thursday, September 30, 2021 8:13 PM Amit Kapila <[email protected]> wrote: > On Thu, Sep 30, 2021 at 1:02 PM Osumi, Takamichi/大墨 昂道 > <[email protected]> wrote: > > > > On Thursday, September 30, 2021 1:15 PM Amit Kapila > <[email protected]> wrote: > > > On Thu, Sep 30, 2021 at 8:22 AM Hou, Zhijie/侯 志杰 > > > <[email protected]> wrote: > > > > > > > > On Tues, Sep 28, 2021 6:05 PM Amit Kapila > > > > <[email protected]> > > > wrote: > > > > Adding a new view seems resonalble, but it will bring another > > > > subscription related view which might be too much. OTOH, I can see > > > > there are already some different views[1] including xact stat, > > > > maybe adding > > > another one is accepatble ? > > > These all views are related to untransmitted to the collector but > > > what we really need is a view similar to pg_stat_archiver or > > > pg_stat_bgwriter which gives information about background workers. > > > Now, the problem as I see is if we go that route then > > > pg_stat_subscription will no longer remain dynamic view and one > > > might consider that as a compatibility break. The other idea I have > > > shared is that we display these stats under the new view introduced > > > by Sawada-San's patch [1] and probably rename that view as > > > pg_stat_subscription_worker where all the stats (xact info and last > > > failure > > > information) about each worker will be displayed. > > Sorry, all the stats in pg_stat_subscription_worker view ? > > > > There was a discussion that > > the xact info should be displayed from pg_stat_subscription with > > existing stats in the same (which will be changed to persist), but > > when your above proposal comes true, the list of > > pg_stat_subscription_worker's columns will be something like below > > (when I list up major columns). > > > > - subid, subrelid and some other relation attributes required > > - 5 stats values moved from pg_stat_subscription > > received_lsn, last_msg_send_time, last_msg_receipt_time, > > latest_end_lsn, latest_end_time > > > > If we go with the view as pg_stat_subscription_worker, we don't need to move > the existing stats of pg_stat_subscription into it. Thank you for clarifying this point. I understand. Best Regards, Takamichi Osumi ^ permalink raw reply [nested|flat] 113+ messages in thread
* RE: Failed transaction statistics to measure the logical replication progress @ 2021-10-14 03:53 [email protected] <[email protected]> parent: Amit Kapila <[email protected]> 1 sibling, 2 replies; 113+ messages in thread From: [email protected] @ 2021-10-14 03:53 UTC (permalink / raw) To: Amit Kapila <[email protected]>; +Cc: [email protected] <[email protected]>; [email protected] <[email protected]>; Masahiko Sawada <[email protected]> On Thursday, September 30, 2021 12:15 PM Amit Kapila <[email protected]> > On Thu, Sep 30, 2021 at 8:22 AM Hou, Zhijie/侯 志杰 <[email protected]> wrote: > > > > On Tues, Sep 28, 2021 6:05 PM Amit Kapila <[email protected]> wrote: > > > > > > Can't we keep the current and new stats both in-memory and persist on > > > disk? So, the persistent stats data will be used to fill the in-memory > > > counters after restarting of workers, otherwise, we will always refer > > > to in-memory values. > > > > I think this approach works, but I have another concern about it. > > > > The current pg_stat_subscription view is listed as "Dynamic Statistics Views" > in > > the document, the data in it seems about the worker process, and the view > data > > shows what the current worker did. But if we keep the new xact stat persist, > > then it's not what the current worker did, it looks more related to the > > subscription historic data. > > > > I see your point. > > > Adding a new view seems resonalble, but it will bring another subscription > > related view which might be too much. OTOH, I can see there are already > some > > different views[1] including xact stat, maybe adding another one is > accepatble ? > > > > These all views are related to untransmitted to the collector but what > we really need is a view similar to pg_stat_archiver or > pg_stat_bgwriter which gives information about background workers. > Now, the problem as I see is if we go that route then > pg_stat_subscription will no longer remain dynamic view and one might > consider that as a compatibility break. The other idea I have shared > is that we display these stats under the new view introduced by > Sawada-San's patch [1] and probably rename that view as > pg_stat_subscription_worker where all the stats (xact info and last > failure information) about each worker will be displayed. Do you have > any opinion on that idea or do you see any problem with it? Personally, I think it seems reasonable to merge the xact stat into the view from sawada-san's patch. One problem I noticed is that pg_stat_subscription_error currently have a 'count' column which show how many times the last error happened. The xact stat here also have a similar value 'xact_error'. I think we might need to rename it or merge them into one in some way. Besides, if we decide to merge xact stat into pg_stat_subscription_error, some column seems need to be renamed. Maybe like: error_message => Last_error_message, command=> last_error_command.. Best regards, Hou zj ^ permalink raw reply [nested|flat] 113+ messages in thread
* RE: Failed transaction statistics to measure the logical replication progress @ 2021-10-14 06:13 [email protected] <[email protected]> parent: [email protected] <[email protected]> 1 sibling, 1 reply; 113+ messages in thread From: [email protected] @ 2021-10-14 06:13 UTC (permalink / raw) To: [email protected] <[email protected]>; Amit Kapila <[email protected]>; +Cc: [email protected] <[email protected]>; Masahiko Sawada <[email protected]> On Thursday, October 14, 2021 12:54 PM Hou, Zhijie/侯 志杰 <[email protected]> wrote: > On Thursday, September 30, 2021 12:15 PM Amit Kapila > <[email protected]> > > On Thu, Sep 30, 2021 at 8:22 AM Hou, Zhijie/侯 志杰 > <[email protected]> wrote: > > > > > > On Tues, Sep 28, 2021 6:05 PM Amit Kapila <[email protected]> > wrote: > > > > > > > > Can't we keep the current and new stats both in-memory and persist > > > > on disk? So, the persistent stats data will be used to fill the > > > > in-memory counters after restarting of workers, otherwise, we will > > > > always refer to in-memory values. > > > > > > I think this approach works, but I have another concern about it. > > > > > > The current pg_stat_subscription view is listed as "Dynamic Statistics > Views" > > in > > > the document, the data in it seems about the worker process, and the > > > view > > data > > > shows what the current worker did. But if we keep the new xact stat > > > persist, then it's not what the current worker did, it looks more > > > related to the subscription historic data. > > > > > > > I see your point. > > > > > Adding a new view seems resonalble, but it will bring another > > > subscription related view which might be too much. OTOH, I can see > > > there are already > > some > > > different views[1] including xact stat, maybe adding another one is > > accepatble ? > > > > > > > These all views are related to untransmitted to the collector but what > > we really need is a view similar to pg_stat_archiver or > > pg_stat_bgwriter which gives information about background workers. > > Now, the problem as I see is if we go that route then > > pg_stat_subscription will no longer remain dynamic view and one might > > consider that as a compatibility break. The other idea I have shared > > is that we display these stats under the new view introduced by > > Sawada-San's patch [1] and probably rename that view as > > pg_stat_subscription_worker where all the stats (xact info and last > > failure information) about each worker will be displayed. Do you have > > any opinion on that idea or do you see any problem with it? > > Personally, I think it seems reasonable to merge the xact stat into the view from > sawada-san's patch. > > One problem I noticed is that pg_stat_subscription_error currently have a > 'count' column which show how many times the last error happened. The xact > stat here also have a similar value 'xact_error'. I think we might need to rename > it or merge them into one in some way. > > Besides, if we decide to merge xact stat into pg_stat_subscription_error, some > column seems need to be renamed. Maybe like: > error_message => Last_error_message, command=> last_error_command.. Yeah, we must make them distinguished clearly. I guessed that you are concerned about amount of renaming codes that could be a bit large or you come up with a necessity to consider the all column names of the pg_stat_subscription_worker together all at once in advance. It's because my instant impression is, when we go with the current xact stats column definitions (xact_commit, xact_commit_bytes, xact_error, xact_error_bytes, xact_abort, xact_abort_bytes), the renaming problem can be solved if I write one additional patch or extend the main patch of xact stats to handle renaming. (This can work to keep both threads independent from each other). Did you have some concern that cannot be handled by this way ? Best Regards, Takamichi Osumi ^ permalink raw reply [nested|flat] 113+ messages in thread
* RE: Failed transaction statistics to measure the logical replication progress @ 2021-10-18 02:51 [email protected] <[email protected]> parent: [email protected] <[email protected]> 0 siblings, 1 reply; 113+ messages in thread From: [email protected] @ 2021-10-18 02:51 UTC (permalink / raw) To: [email protected] <[email protected]>; +Cc: [email protected] <[email protected]>; Masahiko Sawada <[email protected]>; Amit Kapila <[email protected]> On Thursday, October 14, 2021 2:13 PM Osumi, Takamichi wrote: > On Thursday, October 14, 2021 12:54 PM Hou, Zhijie<[email protected]> wrote: > > On Thursday, September 30, 2021 12:15 PM Amit Kapila > > > On Thu, Sep 30, 2021 at 8:22 AM Hou, Zhijie/侯 志杰wrote: > > > > > > > > On Tues, Sep 28, 2021 6:05 PM Amit Kapila > > > > <[email protected]> > > wrote: > > > > > > > > > > Can't we keep the current and new stats both in-memory and > > > > > persist on disk? So, the persistent stats data will be used to > > > > > fill the in-memory counters after restarting of workers, > > > > > otherwise, we will always refer to in-memory values. > > > > > > > > I think this approach works, but I have another concern about it. > > > > > > > > The current pg_stat_subscription view is listed as "Dynamic > > > > Statistics Views" in > > > > the document, the data in it seems about the worker process, and > > > > the view data > > > > shows what the current worker did. But if we keep the new xact > > > > stat persist, then it's not what the current worker did, it looks > > > > more related to the subscription historic data. > > > > > > > > > > I see your point. > > > > > > > Adding a new view seems resonalble, but it will bring another > > > > subscription related view which might be too much. OTOH, I can see > > > > there are already some > > > > different views[1] including xact stat, maybe adding another one > > > > is accepatble ? > > > > > > > > > > These all views are related to untransmitted to the collector but > > > what we really need is a view similar to pg_stat_archiver or > > > pg_stat_bgwriter which gives information about background workers. > > > Now, the problem as I see is if we go that route then > > > pg_stat_subscription will no longer remain dynamic view and one > > > might consider that as a compatibility break. The other idea I have > > > shared is that we display these stats under the new view introduced > > > by Sawada-San's patch [1] and probably rename that view as > > > pg_stat_subscription_worker where all the stats (xact info and last > > > failure information) about each worker will be displayed. Do you > > > have any opinion on that idea or do you see any problem with it? > > > > Personally, I think it seems reasonable to merge the xact stat into > > the view from sawada-san's patch. > > > > One problem I noticed is that pg_stat_subscription_error currently > > have a 'count' column which show how many times the last error > > happened. The xact stat here also have a similar value 'xact_error'. I > > think we might need to rename it or merge them into one in some way. > > > > Besides, if we decide to merge xact stat into > > pg_stat_subscription_error, some column seems need to be renamed. > Maybe like: > > error_message => Last_error_message, command=> last_error_command.. > Yeah, we must make them distinguished clearly. > > I guessed that you are concerned about > amount of renaming codes that could be a bit large or you come up with a > necessity to consider the all column names of the pg_stat_subscription_worker > together all at once in advance. > > It's because my instant impression is, > when we go with the current xact stats column definitions (xact_commit, > xact_commit_bytes, xact_error, xact_error_bytes, xact_abort, > xact_abort_bytes), the renaming problem can be solved if I write one > additional patch or extend the main patch of xact stats to handle renaming. > (This can work to keep both threads independent from each other). > > Did you have some concern that cannot be handled by this way ? Hi, Currently, I don't find some unsolvable issues in this approach. Best regards, Hou zj ^ permalink raw reply [nested|flat] 113+ messages in thread
* Re: Failed transaction statistics to measure the logical replication progress @ 2021-10-18 10:03 Amit Kapila <[email protected]> parent: [email protected] <[email protected]> 1 sibling, 2 replies; 113+ messages in thread From: Amit Kapila @ 2021-10-18 10:03 UTC (permalink / raw) To: [email protected] <[email protected]>; +Cc: [email protected] <[email protected]>; [email protected] <[email protected]>; Masahiko Sawada <[email protected]> On Thu, Oct 14, 2021 at 9:23 AM [email protected] <[email protected]> wrote: > > On Thursday, September 30, 2021 12:15 PM Amit Kapila <[email protected]> > > > > These all views are related to untransmitted to the collector but what > > we really need is a view similar to pg_stat_archiver or > > pg_stat_bgwriter which gives information about background workers. > > Now, the problem as I see is if we go that route then > > pg_stat_subscription will no longer remain dynamic view and one might > > consider that as a compatibility break. The other idea I have shared > > is that we display these stats under the new view introduced by > > Sawada-San's patch [1] and probably rename that view as > > pg_stat_subscription_worker where all the stats (xact info and last > > failure information) about each worker will be displayed. Do you have > > any opinion on that idea or do you see any problem with it? > > Personally, I think it seems reasonable to merge the xact stat into the view from > sawada-san's patch. > > One problem I noticed is that pg_stat_subscription_error > currently have a 'count' column which show how many times the last error > happened. The xact stat here also have a similar value 'xact_error'. I think we > might need to rename it or merge them into one in some way. > > Besides, if we decide to merge xact stat into pg_stat_subscription_error, some column > seems need to be renamed. Maybe like: > error_message => Last_error_message, command=> last_error_command.. > Don't you think that keeping the view name as pg_stat_subscription_error would be a bit confusing if it has to display xact_info? Isn't it better to change it to pg_stat_subscription_worker or some other worker-specific generic name? -- With Regards, Amit Kapila. ^ permalink raw reply [nested|flat] 113+ messages in thread
* RE: Failed transaction statistics to measure the logical replication progress @ 2021-10-19 01:41 [email protected] <[email protected]> parent: Amit Kapila <[email protected]> 1 sibling, 0 replies; 113+ messages in thread From: [email protected] @ 2021-10-19 01:41 UTC (permalink / raw) To: Amit Kapila <[email protected]>; +Cc: [email protected] <[email protected]>; [email protected] <[email protected]>; Masahiko Sawada <[email protected]> On Monday, October 18, 2021 6:04 PM Amit Kapila <[email protected]> wrote: > On Thu, Oct 14, 2021 at 9:23 AM [email protected] > <[email protected]> wrote: > > > > On Thursday, September 30, 2021 12:15 PM Amit Kapila > > <[email protected]> > > > > > > These all views are related to untransmitted to the collector but > > > what we really need is a view similar to pg_stat_archiver or > > > pg_stat_bgwriter which gives information about background workers. > > > Now, the problem as I see is if we go that route then > > > pg_stat_subscription will no longer remain dynamic view and one > > > might consider that as a compatibility break. The other idea I have > > > shared is that we display these stats under the new view introduced > > > by Sawada-San's patch [1] and probably rename that view as > > > pg_stat_subscription_worker where all the stats (xact info and last > > > failure information) about each worker will be displayed. Do you > > > have any opinion on that idea or do you see any problem with it? > > > > Personally, I think it seems reasonable to merge the xact stat into > > the view from sawada-san's patch. > > > > One problem I noticed is that pg_stat_subscription_error currently > > have a 'count' column which show how many times the last error > > happened. The xact stat here also have a similar value 'xact_error'. I > > think we might need to rename it or merge them into one in some way. > > > > Besides, if we decide to merge xact stat into > > pg_stat_subscription_error, some column seems need to be renamed. > Maybe like: > > error_message => Last_error_message, command=> last_error_command.. > > > > Don't you think that keeping the view name as pg_stat_subscription_error > would be a bit confusing if it has to display xact_info? Isn't it better to change it > to pg_stat_subscription_worker or some other worker-specific generic name? Yes, I agreed that rename the view to pg_stat_subscription_worker or some other worker-specific generic name is better if we decide to move forward with this approach. Best regards, Hou zj ^ permalink raw reply [nested|flat] 113+ messages in thread
* RE: Failed transaction statistics to measure the logical replication progress @ 2021-10-19 02:51 [email protected] <[email protected]> parent: [email protected] <[email protected]> 0 siblings, 0 replies; 113+ messages in thread From: [email protected] @ 2021-10-19 02:51 UTC (permalink / raw) To: [email protected] <[email protected]>; +Cc: [email protected] <[email protected]>; Masahiko Sawada <[email protected]>; Amit Kapila <[email protected]> On Monday, October 18, 2021 11:52 AM Hou, Zhijie/侯 志杰 <[email protected]> wrote: > On Thursday, October 14, 2021 2:13 PM Osumi, Takamichi wrote: > > On Thursday, October 14, 2021 12:54 PM Hou, > Zhijie<[email protected]> wrote: > > > On Thursday, September 30, 2021 12:15 PM Amit Kapila > > > > On Thu, Sep 30, 2021 at 8:22 AM Hou, Zhijie/侯 志杰wrote: > > > > > > > > > > On Tues, Sep 28, 2021 6:05 PM Amit Kapila > > > > > <[email protected]> > > > wrote: > > > > > > > > > > > > Can't we keep the current and new stats both in-memory and > > > > > > persist on disk? So, the persistent stats data will be used to > > > > > > fill the in-memory counters after restarting of workers, > > > > > > otherwise, we will always refer to in-memory values. > > > > > > > > > > I think this approach works, but I have another concern about it. > > > > > > > > > > The current pg_stat_subscription view is listed as "Dynamic > > > > > Statistics Views" in the document, the data in it seems about > > > > > the worker process, and the view data shows what the current > > > > > worker did. But if we keep the new xact stat persist, then it's > > > > > not what the current worker did, it looks more related to the > > > > > subscription historic data. > > > > > > > > > > > > > I see your point. > > > > > > > > > Adding a new view seems resonalble, but it will bring another > > > > > subscription related view which might be too much. OTOH, I can > > > > > see there are already some different views[1] including xact > > > > > stat, maybe adding another one is accepatble ? > > > > > > > > > > > > > These all views are related to untransmitted to the collector but > > > > what we really need is a view similar to pg_stat_archiver or > > > > pg_stat_bgwriter which gives information about background workers. > > > > Now, the problem as I see is if we go that route then > > > > pg_stat_subscription will no longer remain dynamic view and one > > > > might consider that as a compatibility break. The other idea I > > > > have shared is that we display these stats under the new view > > > > introduced by Sawada-San's patch [1] and probably rename that view > > > > as pg_stat_subscription_worker where all the stats (xact info and > > > > last failure information) about each worker will be displayed. Do > > > > you have any opinion on that idea or do you see any problem with it? > > > > > > Personally, I think it seems reasonable to merge the xact stat into > > > the view from sawada-san's patch. > > > > > > One problem I noticed is that pg_stat_subscription_error currently > > > have a 'count' column which show how many times the last error > > > happened. The xact stat here also have a similar value 'xact_error'. > > > I think we might need to rename it or merge them into one in some way. > > > > > > Besides, if we decide to merge xact stat into > > > pg_stat_subscription_error, some column seems need to be renamed. > > Maybe like: > > > error_message => Last_error_message, command=> > last_error_command.. > > Yeah, we must make them distinguished clearly. > > > > I guessed that you are concerned about amount of renaming codes that > > could be a bit large or you come up with a necessity to consider the > > all column names of the pg_stat_subscription_worker together all at > > once in advance. > > > > It's because my instant impression is, when we go with the current > > xact stats column definitions (xact_commit, xact_commit_bytes, > > xact_error, xact_error_bytes, xact_abort, xact_abort_bytes), the > > renaming problem can be solved if I write one additional patch or > > extend the main patch of xact stats to handle renaming. > > (This can work to keep both threads independent from each other). > > > > Did you have some concern that cannot be handled by this way ? > Hi, > > Currently, I don't find some unsolvable issues in this approach. Okay. Glad to hear that. Then, I can restart my implementation with this direction. Best Regards, Takamichi Osumi ^ permalink raw reply [nested|flat] 113+ messages in thread
* Re: Failed transaction statistics to measure the logical replication progress @ 2021-10-21 01:18 Masahiko Sawada <[email protected]> parent: Amit Kapila <[email protected]> 1 sibling, 2 replies; 113+ messages in thread From: Masahiko Sawada @ 2021-10-21 01:18 UTC (permalink / raw) To: Amit Kapila <[email protected]>; +Cc: [email protected] <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]> On Mon, Oct 18, 2021 at 7:03 PM Amit Kapila <[email protected]> wrote: > > On Thu, Oct 14, 2021 at 9:23 AM [email protected] > <[email protected]> wrote: > > > > On Thursday, September 30, 2021 12:15 PM Amit Kapila <[email protected]> > > > > > > These all views are related to untransmitted to the collector but what > > > we really need is a view similar to pg_stat_archiver or > > > pg_stat_bgwriter which gives information about background workers. > > > Now, the problem as I see is if we go that route then > > > pg_stat_subscription will no longer remain dynamic view and one might > > > consider that as a compatibility break. The other idea I have shared > > > is that we display these stats under the new view introduced by > > > Sawada-San's patch [1] and probably rename that view as > > > pg_stat_subscription_worker where all the stats (xact info and last > > > failure information) about each worker will be displayed. Do you have > > > any opinion on that idea or do you see any problem with it? > > > > Personally, I think it seems reasonable to merge the xact stat into the view from > > sawada-san's patch. > > > > One problem I noticed is that pg_stat_subscription_error > > currently have a 'count' column which show how many times the last error > > happened. The xact stat here also have a similar value 'xact_error'. I think we > > might need to rename it or merge them into one in some way. > > > > Besides, if we decide to merge xact stat into pg_stat_subscription_error, some column > > seems need to be renamed. Maybe like: > > error_message => Last_error_message, command=> last_error_command.. > > > > Don't you think that keeping the view name as > pg_stat_subscription_error would be a bit confusing if it has to > display xact_info? Isn't it better to change it to > pg_stat_subscription_worker or some other worker-specific generic > name? I agree that it'd be better to include xact info to pg_stat_subscription_errors view rather than making pg_stat_subscription a cumulative view. It would be more simple and consistent. The user might want to reset only either error stats or xact stats but we can do that by passing a value to the reset function. For example, pg_stat_reset_subscription_worker(10, 20, ‘error') for resetting only error stats whereas pg_stat_reset_subscription_worker(10, 20, ‘xact’) for resetting only xact stats etc (passing NULL or omitting the third argument means resetting all stats). I'll change the view name in the next version patch. Regards, -- Masahiko Sawada EDB: https://www.enterprisedb.com/ ^ permalink raw reply [nested|flat] 113+ messages in thread
* Re: Failed transaction statistics to measure the logical replication progress @ 2021-10-21 08:34 Amit Kapila <[email protected]> parent: Masahiko Sawada <[email protected]> 1 sibling, 0 replies; 113+ messages in thread From: Amit Kapila @ 2021-10-21 08:34 UTC (permalink / raw) To: Masahiko Sawada <[email protected]>; +Cc: [email protected] <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]> On Thu, Oct 21, 2021 at 6:49 AM Masahiko Sawada <[email protected]> wrote: > > On Mon, Oct 18, 2021 at 7:03 PM Amit Kapila <[email protected]> wrote: > > > > On Thu, Oct 14, 2021 at 9:23 AM [email protected] > > <[email protected]> wrote: > > > > > > On Thursday, September 30, 2021 12:15 PM Amit Kapila <[email protected]> > > > > > > > > These all views are related to untransmitted to the collector but what > > > > we really need is a view similar to pg_stat_archiver or > > > > pg_stat_bgwriter which gives information about background workers. > > > > Now, the problem as I see is if we go that route then > > > > pg_stat_subscription will no longer remain dynamic view and one might > > > > consider that as a compatibility break. The other idea I have shared > > > > is that we display these stats under the new view introduced by > > > > Sawada-San's patch [1] and probably rename that view as > > > > pg_stat_subscription_worker where all the stats (xact info and last > > > > failure information) about each worker will be displayed. Do you have > > > > any opinion on that idea or do you see any problem with it? > > > > > > Personally, I think it seems reasonable to merge the xact stat into the view from > > > sawada-san's patch. > > > > > > One problem I noticed is that pg_stat_subscription_error > > > currently have a 'count' column which show how many times the last error > > > happened. The xact stat here also have a similar value 'xact_error'. I think we > > > might need to rename it or merge them into one in some way. > > > > > > Besides, if we decide to merge xact stat into pg_stat_subscription_error, some column > > > seems need to be renamed. Maybe like: > > > error_message => Last_error_message, command=> last_error_command.. > > > > > > > Don't you think that keeping the view name as > > pg_stat_subscription_error would be a bit confusing if it has to > > display xact_info? Isn't it better to change it to > > pg_stat_subscription_worker or some other worker-specific generic > > name? > > I agree that it'd be better to include xact info to > pg_stat_subscription_errors view rather than making > pg_stat_subscription a cumulative view. It would be more simple and > consistent. > > The user might want to reset only either error stats or xact stats but > we can do that by passing a value to the reset function. For example, > pg_stat_reset_subscription_worker(10, 20, ‘error') for resetting only > error stats whereas pg_stat_reset_subscription_worker(10, 20, ‘xact’) > for resetting only xact stats etc (passing NULL or omitting the third > argument means resetting all stats). > Sounds reasonable. > I'll change the view name in the > next version patch. > Thanks, that will be helpful. -- With Regards, Amit Kapila. ^ permalink raw reply [nested|flat] 113+ messages in thread
* RE: Failed transaction statistics to measure the logical replication progress @ 2021-10-28 14:18 [email protected] <[email protected]> parent: Masahiko Sawada <[email protected]> 1 sibling, 1 reply; 113+ messages in thread From: [email protected] @ 2021-10-28 14:18 UTC (permalink / raw) To: 'Masahiko Sawada' <[email protected]>; Amit Kapila <[email protected]>; +Cc: [email protected] <[email protected]>; [email protected] <[email protected]> On Thursday, October 21, 2021 10:19 AM Masahiko Sawada <[email protected]> wrote: > On Mon, Oct 18, 2021 at 7:03 PM Amit Kapila <[email protected]> > wrote: > > > > On Thu, Oct 14, 2021 at 9:23 AM [email protected] > > <[email protected]> wrote: > > > > > > On Thursday, September 30, 2021 12:15 PM Amit Kapila > > > <[email protected]> > > > > > > > > These all views are related to untransmitted to the collector but > > > > what we really need is a view similar to pg_stat_archiver or > > > > pg_stat_bgwriter which gives information about background workers. > > > > Now, the problem as I see is if we go that route then > > > > pg_stat_subscription will no longer remain dynamic view and one > > > > might consider that as a compatibility break. The other idea I > > > > have shared is that we display these stats under the new view > > > > introduced by Sawada-San's patch [1] and probably rename that view > > > > as pg_stat_subscription_worker where all the stats (xact info and > > > > last failure information) about each worker will be displayed. Do > > > > you have any opinion on that idea or do you see any problem with it? > > > > > > Personally, I think it seems reasonable to merge the xact stat into > > > the view from sawada-san's patch. > > > > > > One problem I noticed is that pg_stat_subscription_error currently > > > have a 'count' column which show how many times the last error > > > happened. The xact stat here also have a similar value 'xact_error'. > > > I think we might need to rename it or merge them into one in some way. > > > > > > Besides, if we decide to merge xact stat into > > > pg_stat_subscription_error, some column seems need to be renamed. > Maybe like: > > > error_message => Last_error_message, command=> > last_error_command.. > > > > > > > Don't you think that keeping the view name as > > pg_stat_subscription_error would be a bit confusing if it has to > > display xact_info? Isn't it better to change it to > > pg_stat_subscription_worker or some other worker-specific generic > > name? > > I agree that it'd be better to include xact info to pg_stat_subscription_errors > view rather than making pg_stat_subscription a cumulative view. It would be > more simple and consistent. ... >I'll change the view name in the next version patch. Thanks a lot, Sawasa-san. I've created a new patch that extends pg_stat_subscription_workers to include other transaction statistics. Note that this patch depends on v18 patch-set in [1] and needs to be after the perl modules' namespace changes conducted recently by commit b3b4d8e68ae83f432f43f035c7eb481ef93e1583. There are other several major changes compared to the previous version. (1) Addressing several streaming transactions running in parallel on the pub that can send unexpected order of partial data demarcated stream start and stream stop. Even if one of them aborts, now bytes calculation should be correct. (2) Updates of stats on the view at either commit prepared or rollback prepared time. This means we don't lost prepared transaction size even after server restart and user can see the results of two phase operation at those timings. (3) Addition of TAP tests. (4) Renames of other existing columns so that those meanings are more apparent in align with other new stats. Some important details are written in the comments of the attached patch. [1] - https://www.postgresql.org/message-id/CAD21AoCTtQgfy57AxB4q8KUOpRH8rkHN%3Ds_9p9Pvno_XoBK5wg%40mail.g... Best Regards, Takamichi Osumi Attachments: [application/octet-stream] extend_xact_stats_of_subscription_worker_v6.patch (55.6K, ../../OSBPR01MB4888125C82102E3343D85BC7ED869@OSBPR01MB4888.jpnprd01.prod.outlook.com/2-extend_xact_stats_of_subscription_worker_v6.patch) download | inline diff: diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml index 8d4941c..284f215 100644 --- a/doc/src/sgml/monitoring.sgml +++ b/doc/src/sgml/monitoring.sgml @@ -629,7 +629,7 @@ postgres 27093 0.0 0.0 30096 2752 ? Ss 11:34 0:00 postgres: ser <row> <entry><structname>pg_stat_subscription_workers</structname><indexterm><primary>pg_stat_subscription_workers</primary></indexterm></entry> - <entry>At least one row per subscription, showing about errors that + <entry>At least one row per subscription, showing about transaction statistics and error summary that occurred on subscription. See <link linkend="monitoring-pg-stat-subscription-workers"> <structname>pg_stat_subscription_workers</structname></link> for details. @@ -3052,9 +3052,9 @@ SELECT pid, wait_event_type, wait_event FROM pg_stat_activity WHERE wait_event i <para> The <structname>pg_stat_subscription_workers</structname> view will contain - one row per subscription error reported by workers applying logical - replication changes and workers handling the initial data copy of the - subscribed tables. + one row per subscription for transaction statistics and summary of the last + error reported by workers applying logical replication changes and workers + handling the initial data copy of the subscribed tables. </para> <table id="pg-stat-subscription-workers" xreflabel="pg_stat_subscription_workers"> @@ -3102,20 +3102,78 @@ SELECT pid, wait_event_type, wait_event FROM pg_stat_activity WHERE wait_event i <row> <entry role="catalog_table_entry"><para role="column_definition"> - <structfield>relid</structfield> <type>oid</type> + <structfield>last_error_relid</structfield> <type>oid</type> </para> <para> OID of the relation that the worker was processing when the - error occurred + last error occurred </para></entry> </row> <row> <entry role="catalog_table_entry"><para role="column_definition"> - <structfield>command</structfield> <type>text</type> + <structfield>xact_commit</structfield> <type>bigint</type> + </para> + <para> + Number of transactions successfully applied in this subscription + </para></entry> + </row> + + <row> + <entry role="catalog_table_entry"><para role="column_definition"> + <structfield>xact_commit_bytes</structfield> <type>bigint</type> + </para> + <para> + Amount of transactions data successfully applied in this subscription + </para></entry> + </row> + + <row> + <entry role="catalog_table_entry"><para role="column_definition"> + <structfield>xact_error</structfield> <type>bigint</type> + </para> + <para> + Number of transactions unsuccessfully applied in this subscription + </para></entry> + </row> + + <row> + <entry role="catalog_table_entry"><para role="column_definition"> + <structfield>xact_error_bytes</structfield> <type>bigint</type> + </para> + <para> + Amount of transactions data unsuccessfully applied in this subscription + </para></entry> + </row> + + <row> + <entry role="catalog_table_entry"><para role="column_definition"> + <structfield>xact_abort</structfield> <type>bigint</type> + </para> + <para> + Number of transactions aborted in this subscription + </para></entry> + </row> + + <row> + <entry role="catalog_table_entry"><para role="column_definition"> + <structfield>xact_abort_bytes</structfield> <type>bigint</type> + </para> + <para> + Amount of transactions data aborted in this subscription. + Increase <literal>logical_decoding_work_mem</literal> on the publisher + to suppress unnecessary consumed network bandwidth or change in memory + of the subscriber, if unexpected amount of rollbacked transactions are + streamed. + </para></entry> + </row> + + <row> + <entry role="catalog_table_entry"><para role="column_definition"> + <structfield>last_error_command</structfield> <type>text</type> </para> <para> - Name of command being applied when the error occurred. This field + Name of last command being applied when the error occurred. This field is always NULL if the error was reported during the initial data copy. </para></entry> @@ -3123,10 +3181,10 @@ SELECT pid, wait_event_type, wait_event FROM pg_stat_activity WHERE wait_event i <row> <entry role="catalog_table_entry"><para role="column_definition"> - <structfield>xid</structfield> <type>xid</type> + <structfield>last_error_xid</structfield> <type>xid</type> </para> <para> - Transaction ID of the publisher node being applied when the error + Transaction ID of the publisher node being applied when the last error occurred. This field is always NULL if the error was reported during the initial data copy. </para></entry> @@ -3134,19 +3192,19 @@ SELECT pid, wait_event_type, wait_event FROM pg_stat_activity WHERE wait_event i <row> <entry role="catalog_table_entry"><para role="column_definition"> - <structfield>count</structfield> <type>uint8</type> + <structfield>last_error_count</structfield> <type>uint8</type> </para> <para> - Number of consecutive times the error occurred + Number of consecutive times the last error occurred </para></entry> </row> <row> <entry role="catalog_table_entry"><para role="column_definition"> - <structfield>error_message</structfield> <type>text</type> + <structfield>last_error_message</structfield> <type>text</type> </para> <para> - The error message + The last error message </para></entry> </row> diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index 95226b8..5f00053 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1263,11 +1263,17 @@ CREATE VIEW pg_stat_subscription_workers AS e.subid, s.subname, e.subrelid, - e.relid, - e.command, - e.xid, - e.count, - e.error_message, + e.xact_commit, + e.xact_commit_bytes, + e.xact_error, + e.xact_error_bytes, + e.xact_abort, + e.xact_abort_bytes, + e.last_error_relid, + e.last_error_command, + e.last_error_xid, + e.last_error_count, + e.last_error_message, e.last_error_time, e.stats_reset FROM (SELECT diff --git a/src/backend/executor/execPartition.c b/src/backend/executor/execPartition.c index 5c723bc..20184c6 100644 --- a/src/backend/executor/execPartition.c +++ b/src/backend/executor/execPartition.c @@ -192,6 +192,16 @@ static void find_matching_subplans_recurse(PartitionPruningData *prunedata, bool initial_prune, Bitmapset **validsubplans); +/* + * PartitionTupleRoutingSize - exported to calculate total data size + * of logical replication mesage apply, because this is one of the + * ApplyExecutionData struct members. + */ +size_t +PartitionTupleRoutingSize(void) +{ + return sizeof(PartitionTupleRouting); +} /* * ExecSetupPartitionTupleRouting - sets up information needed during diff --git a/src/backend/postmaster/pgstat.c b/src/backend/postmaster/pgstat.c index d9a2048..95c8ece 100644 --- a/src/backend/postmaster/pgstat.c +++ b/src/backend/postmaster/pgstat.c @@ -54,6 +54,7 @@ #include "postmaster/fork_process.h" #include "postmaster/interrupt.h" #include "postmaster/postmaster.h" +#include "replication/logicalworker.h" #include "replication/slot.h" #include "replication/walsender.h" #include "storage/backendid.h" @@ -287,6 +288,13 @@ static PgStat_SLRUStats slruStats[SLRU_NUM_ELEMENTS]; static HTAB *replSlotStatHash = NULL; static HTAB *subWorkerStatHash = NULL; +/* Stats of prepared transactions should be displayed + * at either commit prepared or rollback prepared time, even when it's + * after the server restart. We have the apply worker send those statistics + * to the stats collector at exit and the startup process restore those at restart. + */ +static HTAB *subWorkerPreparedXactSizeHash = NULL; + /* * List of OIDs of databases we need to write out. If an entry is InvalidOid, * it means to write only the shared-catalog stats ("DB 0"); otherwise, we @@ -338,6 +346,9 @@ static void pgstat_reset_replslot(PgStat_StatReplSlotEntry *slotstats, Timestamp static PgStat_StatSubWorkerEntry *pgstat_get_subworker_entry(Oid subid, Oid subrelid, bool create); +static PgStat_StatSubWorkerPreparedXactSize *pgstat_get_subworker_prepared_txn(Oid subid, + char *gid, bool create); + static void pgstat_reset_subworker_entry(PgStat_StatSubWorkerEntry *wentry, TimestampTz ts); static void pgstat_report_subscription_purge(PgStat_MsgSubscriptionPurge *msg); static void pgstat_report_subworker_purge(PgStat_MsgSubWorkerPurge *msg); @@ -388,6 +399,9 @@ static void pgstat_recv_tempfile(PgStat_MsgTempFile *msg, int len); static void pgstat_recv_subscription_purge(PgStat_MsgSubscriptionPurge *msg, int len); static void pgstat_recv_subworker_error(PgStat_MsgSubWorkerError *msg, int len); static void pgstat_recv_subworker_purge(PgStat_MsgSubWorkerPurge *msg, int len); +static void pgstat_recv_subworker_xact_end(PgStat_MsgSubWorkerXactEnd *msg, int len); +static void pgstat_recv_subworker_twophase_xact(PgStat_MsgSubWorkerTwophaseXact *msg, int len); + /* ------------------------------------------------------------ * Public functions called from postmaster follow @@ -2115,6 +2129,73 @@ pgstat_report_replslot_drop(const char *slotname) } /* ---------- + * pgstat_report_subworker_xact_end() - + * + * Tell the collector that worker transaction has finished without problem. + * ---------- + */ +void +pgstat_report_subworker_xact_end(Oid subid, Oid subrel, + LogicalRepMsgType command, PgStat_Counter xact_size) +{ + PgStat_MsgSubWorkerXactEnd msg; + + Assert(command == 0 /* table sync worker */ || + command == LOGICAL_REP_MSG_COMMIT || + command == LOGICAL_REP_MSG_STREAM_ABORT || + command == LOGICAL_REP_MSG_STREAM_COMMIT); + + pgstat_setheader(&msg.m_hdr, PGSTAT_MTYPE_SUBWORKERXACTEND); + msg.m_subid = subid; + msg.m_subrelid = subrel; + msg.m_command = command; + msg.m_xact_bytes = xact_size; + pgstat_send(&msg, sizeof(PgStat_MsgSubWorkerXactEnd)); + + reset_apply_error_context_xact_size(); +} + +/* ---------- + * pgstat_report_subworker_twophase_xact() - + * + * Tell the collector that worker transaction has done 2PC related operation. + * ---------- + */ +void +pgstat_report_subworker_twophase_xact(Oid subid, LogicalRepMsgType command, + PgStat_Counter xact_size, + LogicalRepPreparedTxnData *prepare_data, + LogicalRepCommitPreparedTxnData *commit_data, + LogicalRepRollbackPreparedTxnData *rollback_data) +{ + PgStat_MsgSubWorkerTwophaseXact msg; + + Assert(command == LOGICAL_REP_MSG_PREPARE || + command == LOGICAL_REP_MSG_STREAM_PREPARE || + command == LOGICAL_REP_MSG_COMMIT_PREPARED || + command == LOGICAL_REP_MSG_ROLLBACK_PREPARED); + + /* setup the message */ + pgstat_setheader(&msg.m_hdr, PGSTAT_MTYPE_SUBWORKERTWOPHASEXACT); + msg.m_subid = subid; + msg.m_command = command; + + /* get the gid for this two phase operation */ + if (command == LOGICAL_REP_MSG_PREPARE || + command == LOGICAL_REP_MSG_STREAM_PREPARE) + memcpy(msg.m_gid, prepare_data->gid, GIDSIZE); + else if (command == LOGICAL_REP_MSG_COMMIT_PREPARED) + memcpy(msg.m_gid, commit_data->gid, GIDSIZE); + else /* rollback prepared */ + memcpy(msg.m_gid, rollback_data->gid, GIDSIZE); + + msg.m_xact_bytes = xact_size; + pgstat_send(&msg, sizeof(PgStat_MsgSubWorkerTwophaseXact)); + + reset_apply_error_context_xact_size(); +} + +/* ---------- * pgstat_report_subworker_error() - * * Tell the collector about the subscription worker error. @@ -2134,6 +2215,7 @@ pgstat_report_subworker_error(Oid subid, Oid subrelid, Oid relid, pgstat_setheader(&msg.m_hdr, PGSTAT_MTYPE_SUBWORKERERROR); msg.m_subid = subid; msg.m_subrelid = subrelid; + msg.m_xact_error_bytes = get_apply_error_context_xact_size(); msg.m_dbid = MyDatabaseId; msg.m_relid = relid; msg.m_command = command; @@ -2142,6 +2224,8 @@ pgstat_report_subworker_error(Oid subid, Oid subrelid, Oid relid, strlcpy(msg.m_message, errmsg, PGSTAT_SUBWORKERERROR_MSGLEN); pgstat_send(&msg, len); + + reset_apply_error_context_xact_size(); } /* ---------- @@ -3876,6 +3960,14 @@ PgstatCollectorMain(int argc, char *argv[]) pgstat_recv_subworker_purge(&msg.msg_subworkerpurge, len); break; + case PGSTAT_MTYPE_SUBWORKERXACTEND: + pgstat_recv_subworker_xact_end(&msg.msg_subworkerxactend, len); + break; + + case PGSTAT_MTYPE_SUBWORKERTWOPHASEXACT: + pgstat_recv_subworker_twophase_xact(&msg.msg_subworkertwophasexact, len); + break; + default: break; } @@ -4193,6 +4285,22 @@ pgstat_write_statsfiles(bool permanent, bool allDbs) } /* + * Write subscription worker's prepared transaction struct + */ + if (subWorkerPreparedXactSizeHash) + { + PgStat_StatSubWorkerPreparedXactSize *prepared_size; + + hash_seq_init(&hstat, subWorkerPreparedXactSizeHash); + while((prepared_size = (PgStat_StatSubWorkerPreparedXactSize *) hash_seq_search(&hstat)) != NULL) + { + fputc('P', fpout); + rc = fwrite(prepared_size, sizeof(PgStat_StatSubWorkerPreparedXactSize), 1, fpout); + (void) rc; /* we'll check for error with ferror */ + } + } + + /* * No more output to be done. Close the temp file and replace the old * pgstat.stat with it. The ferror() check replaces testing for error * after each individual fputc or fwrite above. @@ -4695,6 +4803,40 @@ pgstat_read_statsfiles(Oid onlydb, bool permanent, bool deep) break; } + case 'P': + { + PgStat_StatSubWorkerPreparedXactSize buff; + PgStat_StatSubWorkerPreparedXactSize *prepared_xact_size; + + if (fread(&buff, 1, sizeof(PgStat_StatSubWorkerPreparedXactSize), + fpin) != sizeof(PgStat_StatSubWorkerPreparedXactSize)) + { + ereport(pgStatRunningInCollector ? LOG : WARNING, + (errmsg("corrupted statistics file \"%s\"", + statfile))); + goto done; + } + + if (subWorkerPreparedXactSizeHash == NULL) + { + HASHCTL hash_ctl; + + hash_ctl.keysize = sizeof(PgStat_StatSubWorkerPreparedXact); + hash_ctl.entrysize = sizeof(PgStat_StatSubWorkerPreparedXactSize); + hash_ctl.hcxt = pgStatLocalContext; + subWorkerPreparedXactSizeHash = hash_create("Subscription worker stats of prepared txn", + PGSTAT_SUBWORKER_HASH_SIZE, + &hash_ctl, + HASH_ELEM | HASH_STRINGS | HASH_CONTEXT); + } + + prepared_xact_size = + (PgStat_StatSubWorkerPreparedXactSize *) hash_search(subWorkerPreparedXactSizeHash, + (void *) &buff.key, + HASH_ENTER, NULL); + memcpy(prepared_xact_size, &buff, sizeof(PgStat_StatSubWorkerPreparedXactSize)); + break; + } case 'E': goto done; @@ -5260,6 +5402,7 @@ pgstat_clear_snapshot(void) pgStatDBHash = NULL; replSlotStatHash = NULL; subWorkerStatHash = NULL; + subWorkerPreparedXactSizeHash = NULL; /* * Historically the backend_status.c facilities lived in this file, and @@ -6259,6 +6402,105 @@ pgstat_recv_subscription_purge(PgStat_MsgSubscriptionPurge *msg, int len) } /* ---------- + * pgstat_recv_subworker_xact_end() - + * + * Process a SUBWORKERXACTEND message. + * ---------- + */ +static void +pgstat_recv_subworker_xact_end(PgStat_MsgSubWorkerXactEnd *msg, int len) +{ + PgStat_StatSubWorkerEntry *wentry; + + wentry = pgstat_get_subworker_entry(msg->m_subid, msg->m_subrelid, true); + Assert(wentry); + + /* table sync worker */ + if (msg->m_command == 0) + wentry->xact_commit++; + else + { + /* apply worker */ + switch(msg->m_command) + { + case LOGICAL_REP_MSG_COMMIT: + case LOGICAL_REP_MSG_STREAM_COMMIT: + wentry->xact_commit++; + wentry->xact_commit_bytes += msg->m_xact_bytes; + break; + case LOGICAL_REP_MSG_STREAM_ABORT: + wentry->xact_abort++; + wentry->xact_abort_bytes += msg->m_xact_bytes; + break; + default: + elog(ERROR, "unexpected logical message type as normal apply end"); + break; + } + } +} + +/* ---------- + * pgstat_recv_subworker_twophase_xact() - + * + * Process a SUBWORKERTWOPHASEXACT message. + * ---------- + */ +static void +pgstat_recv_subworker_twophase_xact(PgStat_MsgSubWorkerTwophaseXact *msg, int len) +{ + PgStat_StatSubWorkerPreparedXactSize *prepared_txn; + PgStat_StatSubWorkerEntry *wentry; + PgStat_StatSubWorkerPreparedXact key; + + prepared_txn = pgstat_get_subworker_prepared_txn(msg->m_subid, msg->m_gid, true); + Assert(prepared_txn); + switch(msg->m_command) + { + case LOGICAL_REP_MSG_PREPARE: + case LOGICAL_REP_MSG_STREAM_PREPARE: + /* + * Make each size of prepared transaction persistent + * so that we can update stats over the server restart + * and make prepared stats updated when commit prepared + * or rollback prepared arrives. + */ + prepared_txn->subid = msg->m_subid; + strlcpy(prepared_txn->gid, msg->m_gid, strlen(msg->m_gid) + 1); + prepared_txn->xact_size = msg->m_xact_bytes; + break; + + case LOGICAL_REP_MSG_COMMIT_PREPARED: + case LOGICAL_REP_MSG_ROLLBACK_PREPARED: + /* Update exported xact stats now */ + wentry = pgstat_get_subworker_entry(msg->m_subid, + InvalidOid /* apply worker */, + true); + Assert(wentry); + if (msg->m_command == LOGICAL_REP_MSG_COMMIT_PREPARED) + { + wentry->xact_commit++; + wentry->xact_commit_bytes += prepared_txn->xact_size; + } + else + { + wentry->xact_abort++; + wentry->xact_abort_bytes += prepared_txn->xact_size; + } + + /* Clean up this gid from transaction size hash */ + key.subid = prepared_txn->subid; + memcpy(key.gid, msg->m_gid, strlen(msg->m_gid)); + (void) hash_search(subWorkerPreparedXactSizeHash, + (void *) &key, HASH_REMOVE, NULL); + break; + + default: + elog(ERROR, "unexpected logical message type as prepare transaction"); + break; + } +} + +/* ---------- * pgstat_recv_subworker_error() - * * Process a SUBWORKERERROR message. @@ -6273,6 +6515,10 @@ pgstat_recv_subworker_error(PgStat_MsgSubWorkerError *msg, int len) wentry = pgstat_get_subworker_entry(msg->m_subid, msg->m_subrelid, true); Assert(wentry); + /* general transaction stats for error */ + wentry->xact_error++; + wentry->xact_error_bytes += msg->m_xact_error_bytes; + /* * Update only the counter and timestamp if we received the same error * again @@ -6489,6 +6735,51 @@ pgstat_get_subworker_entry(Oid subid, Oid subrelid, bool create) } /* ---------- + * pgstat_get_subworker_prepared_txn + * + * Return subscription worker entry with the given subscription OID and + * gid. + * ---------- + */ +static PgStat_StatSubWorkerPreparedXactSize* +pgstat_get_subworker_prepared_txn(Oid subid, char *gid, bool create) +{ + PgStat_StatSubWorkerPreparedXact key; + PgStat_StatSubWorkerPreparedXactSize *prepared_txn_size; + HASHACTION action; + bool found; + + if (subWorkerPreparedXactSizeHash == NULL) + { + HASHCTL hash_ctl; + + hash_ctl.keysize = sizeof(PgStat_StatSubWorkerPreparedXact); + hash_ctl.entrysize = sizeof(PgStat_StatSubWorkerPreparedXactSize); + hash_ctl.hcxt = pgStatLocalContext; + subWorkerPreparedXactSizeHash = hash_create("Subscription worker stats of prepared txn", + PGSTAT_SUBWORKER_HASH_SIZE, + &hash_ctl, + HASH_ELEM | HASH_STRINGS); + } + + key.subid = subid; + memcpy(key.gid, gid, strlen(gid)); + action = (create ? HASH_ENTER : HASH_FIND); + prepared_txn_size = (PgStat_StatSubWorkerPreparedXactSize *) hash_search(subWorkerPreparedXactSizeHash, + (void *) &key, + action, &found); + + if (create && !found) + { + prepared_txn_size->subid = 0; + prepared_txn_size->gid[0] = '\0'; + prepared_txn_size->xact_size = 0; + } + + return prepared_txn_size; +} + +/* ---------- * pgstat_reset_subworker_entry * * Reset the given subscription worker statistics. diff --git a/src/backend/replication/logical/tablesync.c b/src/backend/replication/logical/tablesync.c index f07983a..aff78fd 100644 --- a/src/backend/replication/logical/tablesync.c +++ b/src/backend/replication/logical/tablesync.c @@ -1149,6 +1149,12 @@ copy_table_done: MyLogicalRepWorker->relstate_lsn = *origin_startpos; SpinLockRelease(&MyLogicalRepWorker->relmutex); + /* Report the success of table sync. */ + pgstat_report_subworker_xact_end(MyLogicalRepWorker->subid, + MyLogicalRepWorker->relid, + 0 /* no logical message type */, + 0 /* xact size */); + /* * Finally, wait until the main apply worker tells us to catch up and then * return to let LogicalRepApplyLoop do it. diff --git a/src/backend/replication/logical/worker.c b/src/backend/replication/logical/worker.c index d771a0c..6284310 100644 --- a/src/backend/replication/logical/worker.c +++ b/src/backend/replication/logical/worker.c @@ -222,6 +222,21 @@ typedef struct ApplyErrorCallbackArg LogicalRepMsgType command; /* 0 if invalid */ LogicalRepRelMapEntry *rel; + /* + * Store data size of this transaction. + * + * The byte size of transaction on the publisher is calculated + * by ReorderBufferChangeSize() based on the ReorderBufferChange + * structure. But on the subscriber, consumed resources are + * not same as the publisher's decoding processsing and required + * to be computed in different way. Therefore, the exact same byte + * size is not restored on the subscriber usually. + * + * Data size of streaming transactions is managed by streamingXactSize + * for flexible data blocks handling. + */ + PgStat_Counter bytes; + /* Remote node information */ int remote_attnum; /* -1 if invalid */ TransactionId remote_xid; @@ -232,11 +247,32 @@ static ApplyErrorCallbackArg apply_error_callback_arg = { .command = 0, .rel = NULL, + .bytes = 0, .remote_attnum = -1, .remote_xid = InvalidTransactionId, .ts = 0, }; +/* + * Two or more streaming transactions in parallel on the publisher + * generate unexpected order of partial txn data demarcated stream start + * and stream stop to the subscriber, since whenever its size of one of + * the txns reaches the publisher's logical_decoding_work_mem, + * the part (and in the end, last remaining changes) is streamed. + * This creates mixed blocks of streaming data while + * there's possibility some are successfully committed but others are + * not by stream abort. Therefore, to track correct byte size + * it's necessary to trace each streaming transaction by making paring + * of xid and transaction size. + */ +#define PARALLEL_STREAMING_XACTS 32 +typedef struct XactSizeEntry +{ + TransactionId key; + PgStat_Counter xact_size; +} XactSizeEntry; +static HTAB *streamingXactSize = NULL; + static MemoryContext ApplyMessageContext = NULL; MemoryContext ApplyContext = NULL; @@ -320,6 +356,8 @@ static void maybe_reread_subscription(void); /* prototype needed because of stream_commit */ static void apply_dispatch(StringInfo s); +void update_apply_change_size(LogicalRepMsgType action, void *data); + static void apply_handle_commit_internal(LogicalRepCommitData *commit_data); static void apply_handle_insert_internal(ApplyExecutionData *edata, ResultRelInfo *relinfo, @@ -853,6 +891,11 @@ apply_handle_commit(StringInfo s) /* Process any tables that are being synchronized in parallel. */ process_syncing_tables(commit_data.end_lsn); + pgstat_report_subworker_xact_end(MyLogicalRepWorker->subid, + InvalidOid, + LOGICAL_REP_MSG_COMMIT, + get_apply_error_context_xact_size()); + pgstat_report_activity(STATE_IDLE, NULL); reset_apply_error_context_info(); } @@ -965,6 +1008,11 @@ apply_handle_prepare(StringInfo s) /* Process any tables that are being synchronized in parallel. */ process_syncing_tables(prepare_data.end_lsn); + pgstat_report_subworker_twophase_xact(MyLogicalRepWorker->subid, + LOGICAL_REP_MSG_PREPARE, + get_apply_error_context_xact_size(), + &prepare_data, NULL, NULL); + pgstat_report_activity(STATE_IDLE, NULL); reset_apply_error_context_info(); } @@ -1006,6 +1054,13 @@ apply_handle_commit_prepared(StringInfo s) /* Process any tables that are being synchronized in parallel. */ process_syncing_tables(prepare_data.end_lsn); + update_apply_change_size(LOGICAL_REP_MSG_COMMIT_PREPARED, NULL); + + pgstat_report_subworker_twophase_xact(MyLogicalRepWorker->subid, + LOGICAL_REP_MSG_COMMIT_PREPARED, + get_apply_error_context_xact_size(), + NULL, &prepare_data, NULL); + pgstat_report_activity(STATE_IDLE, NULL); reset_apply_error_context_info(); } @@ -1057,6 +1112,12 @@ apply_handle_rollback_prepared(StringInfo s) /* Process any tables that are being synchronized in parallel. */ process_syncing_tables(rollback_data.rollback_end_lsn); + /* send rollback prepared message for this gid */ + pgstat_report_subworker_twophase_xact(MyLogicalRepWorker->subid, + LOGICAL_REP_MSG_ROLLBACK_PREPARED, + get_apply_error_context_xact_size(), + NULL, NULL, &rollback_data); + pgstat_report_activity(STATE_IDLE, NULL); reset_apply_error_context_info(); } @@ -1072,6 +1133,7 @@ static void apply_handle_stream_prepare(StringInfo s) { LogicalRepPreparedTxnData prepare_data; + XactSizeEntry *streamed_entry; if (in_streamed_transaction) ereport(ERROR, @@ -1116,6 +1178,19 @@ apply_handle_stream_prepare(StringInfo s) /* Process any tables that are being synchronized in parallel. */ process_syncing_tables(prepare_data.end_lsn); + /* + * Report the prepared streaming xact size to the stats collector + * in a prepared xact manner to make it survive over the restart. + */ + streamed_entry = hash_search(streamingXactSize, + (void *) &prepare_data.xid, + HASH_FIND, NULL); + Assert(streamed_entry); + pgstat_report_subworker_twophase_xact(MyLogicalRepWorker->subid, + LOGICAL_REP_MSG_STREAM_PREPARE, + streamed_entry->xact_size, + &prepare_data, NULL, NULL); + pgstat_report_activity(STATE_IDLE, NULL); reset_apply_error_context_info(); @@ -1194,6 +1269,8 @@ apply_handle_stream_start(StringInfo s) MyLogicalRepWorker->stream_fileset = palloc(sizeof(FileSet)); FileSetInit(MyLogicalRepWorker->stream_fileset); + update_apply_change_size(LOGICAL_REP_MSG_STREAM_START, NULL); + MemoryContextSwitchTo(oldctx); } @@ -1244,12 +1321,17 @@ apply_handle_stream_stop(StringInfo s) /* * Handle STREAM abort message. + * + * Currently, abort of streaming subtransaction does not affect + * size of streaming transaction resources because it have used the + * resources anyway. */ static void apply_handle_stream_abort(StringInfo s) { TransactionId xid; TransactionId subxid; + XactSizeEntry *streamed_entry; if (in_streamed_transaction) ereport(ERROR, @@ -1265,8 +1347,36 @@ apply_handle_stream_abort(StringInfo s) */ if (xid == subxid) { + bool found = false; set_apply_error_context_xact(xid, 0); stream_cleanup_files(MyLogicalRepWorker->subid, xid); + + /* + * We've completed to handle stream abort without issue, so + * get ready to report the transaction stats via normal + * termination route instead of the apply error route. + */ + streamed_entry = hash_search(streamingXactSize, + (void *) &xid, + HASH_FIND, &found); + /* + * It's possible that we get stream abort + * earlier than any call of write_stream_change that + * creates one hash entry for this xid. In this case, + * to find a entry with this xid fails. So just check + * if we've found it. Only when we confirm some writes + * by write_stream_change, report the stream_abort. + */ + if (found) + { + pgstat_report_subworker_xact_end(MyLogicalRepWorker->subid, + InvalidOid, + LOGICAL_REP_MSG_STREAM_ABORT, + streamed_entry->xact_size); + (void) hash_search(streamingXactSize, + (void *) &xid, + HASH_REMOVE, NULL); + } } else { @@ -1473,6 +1583,7 @@ apply_handle_stream_commit(StringInfo s) { TransactionId xid; LogicalRepCommitData commit_data; + XactSizeEntry *streamed_entry; if (in_streamed_transaction) ereport(ERROR, @@ -1508,6 +1619,19 @@ apply_handle_stream_commit(StringInfo s) /* Process any tables that are being synchronized in parallel. */ process_syncing_tables(commit_data.end_lsn); + /* Report and clean up the xid */ + streamed_entry = hash_search(streamingXactSize, + (void *) &xid, + HASH_FIND, NULL); + Assert(streamed_entry); + pgstat_report_subworker_xact_end(MyLogicalRepWorker->subid, + InvalidOid, + LOGICAL_REP_MSG_STREAM_COMMIT, + streamed_entry->xact_size); + (void) hash_search(streamingXactSize, + (void *) &xid, + HASH_REMOVE, NULL); + pgstat_report_activity(STATE_IDLE, NULL); reset_apply_error_context_info(); @@ -1646,6 +1770,8 @@ apply_handle_insert(StringInfo s) slot_fill_defaults(rel, estate, remoteslot); MemoryContextSwitchTo(oldctx); + update_apply_change_size(LOGICAL_REP_MSG_INSERT, rel); + /* For a partitioned table, insert the tuple into a partition. */ if (rel->localrel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE) apply_handle_tuple_routing(edata, @@ -1803,6 +1929,8 @@ apply_handle_update(StringInfo s) has_oldtup ? &oldtup : &newtup); MemoryContextSwitchTo(oldctx); + update_apply_change_size(LOGICAL_REP_MSG_UPDATE, rel); + /* For a partitioned table, apply update to correct partition. */ if (rel->localrel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE) apply_handle_tuple_routing(edata, @@ -1937,6 +2065,8 @@ apply_handle_delete(StringInfo s) slot_store_data(remoteslot, rel, &oldtup); MemoryContextSwitchTo(oldctx); + update_apply_change_size(LOGICAL_REP_MSG_DELETE, rel); + /* For a partitioned table, apply delete to correct partition. */ if (rel->localrel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE) apply_handle_tuple_routing(edata, @@ -2499,6 +2629,117 @@ apply_dispatch(StringInfo s) } /* + * Subscriber side implementation equivalent to ReorderBufferChangeSize + * of the publisher. + * + * According to the logical replication message type, record major + * resource consumptions of this subscription for each message. + * At present, do not collect data from generic functions to keep + * code simplicity, since the implementation complexity versus benefit + * tradeoff should not be good. Also, add multiple values + * at once in order to reduce the number of this function call. + * + * 'data' controls detail handling of data size calculation. + */ +void +update_apply_change_size(LogicalRepMsgType action, void *data) +{ + int64 size = 0; + LogicalRepRelMapEntry *relmapentry; + LogicalRepRelation *reprelation; + int *stream_write_len; + + /* + * In streaming mode, stream_write_change is called + * instead of immediate apply. List up the messages types + * that can be caught by handle_streamed_transaction and + * treat the write length as the size of transaction so + * that we can export it as part of pg_stat_subscription_worker. + */ + if (in_streamed_transaction && + (action == LOGICAL_REP_MSG_INSERT || + action == LOGICAL_REP_MSG_UPDATE || + action == LOGICAL_REP_MSG_DELETE || + action == LOGICAL_REP_MSG_TRUNCATE || + action == LOGICAL_REP_MSG_RELATION || + action == LOGICAL_REP_MSG_TYPE)) + { + stream_write_len = (int *) data; + size += *stream_write_len; + add_apply_error_context_xact_size(size); + return; + } + + switch (action) + { + /* No special memory consumption */ + case LOGICAL_REP_MSG_BEGIN: + case LOGICAL_REP_MSG_COMMIT: + case LOGICAL_REP_MSG_TRUNCATE: + case LOGICAL_REP_MSG_TYPE: + case LOGICAL_REP_MSG_ORIGIN: + case LOGICAL_REP_MSG_MESSAGE: + case LOGICAL_REP_MSG_STREAM_STOP: + case LOGICAL_REP_MSG_STREAM_ABORT: + case LOGICAL_REP_MSG_BEGIN_PREPARE: + break; + + case LOGICAL_REP_MSG_INSERT: + case LOGICAL_REP_MSG_UPDATE: + case LOGICAL_REP_MSG_DELETE: + Assert(data != NULL); + + /* + * Compute size based on ApplyExecutionData. + * The size of LogicalRepRelMapEntry can be skipped because + * it is obtained from hash_search in logicalrep_rel_open. + */ + size += sizeof(ApplyExecutionData) + sizeof(EState) + + sizeof(ResultRelInfo) + sizeof(ResultRelInfo); + + /* + * Add some extra size if the target relation is partitioned. + * PartitionTupleRouting isn't exported. Therefore, call the + * function that returns its size instead. + */ + relmapentry = (LogicalRepRelMapEntry *) data; + if (relmapentry->localrel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE) + size += sizeof(ModifyTableState) + PartitionTupleRoutingSize(); + break; + + case LOGICAL_REP_MSG_RELATION: + Assert(data != NULL); + + reprelation = (LogicalRepRelation *) data; + /* See logicalrep_read_attrs for the last two */ + size += sizeof(LogicalRepRelation) + + reprelation->natts * sizeof(char *) + + reprelation->natts * sizeof(Oid); + break; + + case LOGICAL_REP_MSG_STREAM_START: + size += sizeof(FileSet); + break; + + case LOGICAL_REP_MSG_PREPARE: + case LOGICAL_REP_MSG_COMMIT_PREPARED: + case LOGICAL_REP_MSG_ROLLBACK_PREPARED: + case LOGICAL_REP_MSG_STREAM_COMMIT: + case LOGICAL_REP_MSG_STREAM_PREPARE: + size += sizeof(FlushPosition); + break; + + default: + ereport(ERROR, + (errcode(ERRCODE_PROTOCOL_VIOLATION), + errmsg("invalid logical replication message type \"%c\"", action))); + } + + /* update the total size of consumption */ + add_apply_error_context_xact_size(size); +} + +/* * Figure out which write/flush positions to report to the walsender process. * * We can't simply report back the last LSN the walsender sent us because the @@ -3352,6 +3593,9 @@ static void stream_write_change(char action, StringInfo s) { int len; + int total_len; + bool found; + XactSizeEntry *streamed_entry; Assert(in_streamed_transaction); Assert(TransactionIdIsValid(stream_xid)); @@ -3370,6 +3614,16 @@ stream_write_change(char action, StringInfo s) len = (s->len - s->cursor); BufFileWrite(stream_fd, &s->data[s->cursor], len); + + /* update xact size by xid */ + total_len = (s->len - s->cursor) * 2 + sizeof(char) + sizeof(action); + streamed_entry = (XactSizeEntry *) hash_search(streamingXactSize, + (void *) &stream_xid, + HASH_ENTER, &found); + if (!found) + streamed_entry->xact_size = total_len; /* init */ + else + streamed_entry->xact_size = streamed_entry->xact_size + total_len; /* update */ } /* @@ -3507,6 +3761,23 @@ ApplyWorkerMain(Datum main_arg) elog(DEBUG1, "connecting to publisher using connection string \"%s\"", MySubscription->conninfo); + /* + * Initialize the apply worker's hash to manage bytes for + * streaming txns. + */ + if (!am_tablesync_worker() && MySubscription->stream) + { + HASHCTL hash_ctl; + + hash_ctl.keysize = sizeof(TransactionId); + hash_ctl.entrysize = sizeof(XactSizeEntry); + hash_ctl.hcxt = LogicalStreamingContext; + streamingXactSize = hash_create("xact size per streaming xid", + PARALLEL_STREAMING_XACTS, + &hash_ctl, + HASH_ELEM | HASH_BLOBS | HASH_CONTEXT); + } + if (am_tablesync_worker()) { char *syncslotname; @@ -3747,6 +4018,27 @@ reset_apply_error_context_info(void) set_apply_error_context_xact(InvalidTransactionId, 0); } +/* Exported so that stats collector can utilize this value */ +int64 +get_apply_error_context_xact_size(void) +{ + return apply_error_callback_arg.bytes; +} + +/* Add size to apply error bytes */ +void +add_apply_error_context_xact_size(int64 size) +{ + apply_error_callback_arg.bytes += size; +} + +/* Reset information of apply error callback */ +void +reset_apply_error_context_xact_size(void) +{ + apply_error_callback_arg.bytes = 0; +} + /* * Start skipping changes of the transaction if the given XID matches the * transaction ID specified by skip_xid option. diff --git a/src/backend/utils/adt/pgstatfuncs.c b/src/backend/utils/adt/pgstatfuncs.c index b58a61d..47eb6da 100644 --- a/src/backend/utils/adt/pgstatfuncs.c +++ b/src/backend/utils/adt/pgstatfuncs.c @@ -2405,7 +2405,7 @@ pg_stat_get_replication_slot(PG_FUNCTION_ARGS) Datum pg_stat_get_subscription_worker(PG_FUNCTION_ARGS) { -#define PG_STAT_GET_SUBSCRIPTION_WORKER_COLS 9 +#define PG_STAT_GET_SUBSCRIPTION_WORKER_COLS 15 Oid subid = PG_GETARG_OID(0); Oid subrelid; TupleDesc tupdesc; @@ -2425,19 +2425,31 @@ pg_stat_get_subscription_worker(PG_FUNCTION_ARGS) OIDOID, -1, 0); TupleDescInitEntry(tupdesc, (AttrNumber) 2, "subrelid", OIDOID, -1, 0); - TupleDescInitEntry(tupdesc, (AttrNumber) 3, "relid", + TupleDescInitEntry(tupdesc, (AttrNumber) 3, "xact_commit", + INT8OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) 4, "xact_commit_bytes", + INT8OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) 5, "xact_error", + INT8OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) 6, "xact_error_bytes", + INT8OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) 7, "xact_abort", + INT8OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) 8, "xact_abort_bytes", + INT8OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) 9, "last_error_relid", OIDOID, -1, 0); - TupleDescInitEntry(tupdesc, (AttrNumber) 4, "command", + TupleDescInitEntry(tupdesc, (AttrNumber) 10, "last_error_command", TEXTOID, -1, 0); - TupleDescInitEntry(tupdesc, (AttrNumber) 5, "xid", + TupleDescInitEntry(tupdesc, (AttrNumber) 11, "last_error_xid", XIDOID, -1, 0); - TupleDescInitEntry(tupdesc, (AttrNumber) 6, "count", + TupleDescInitEntry(tupdesc, (AttrNumber) 12, "last_error_count", INT8OID, -1, 0); - TupleDescInitEntry(tupdesc, (AttrNumber) 7, "error_message", + TupleDescInitEntry(tupdesc, (AttrNumber) 13, "last_error_message", TEXTOID, -1, 0); - TupleDescInitEntry(tupdesc, (AttrNumber) 8, "last_error_time", + TupleDescInitEntry(tupdesc, (AttrNumber) 14, "last_error_time", TIMESTAMPTZOID, -1, 0); - TupleDescInitEntry(tupdesc, (AttrNumber) 9, "stats_reset", + TupleDescInitEntry(tupdesc, (AttrNumber) 15, "stats_reset", TIMESTAMPTZOID, -1, 0); BlessTupleDesc(tupdesc); @@ -2462,28 +2474,36 @@ pg_stat_get_subscription_worker(PG_FUNCTION_ARGS) else nulls[i++] = true; - /* relid */ + /* transaction stats */ + values[i++] = Int64GetDatum(wentry->xact_commit); + values[i++] = Int64GetDatum(wentry->xact_commit_bytes); + values[i++] = Int64GetDatum(wentry->xact_error); + values[i++] = Int64GetDatum(wentry->xact_error_bytes); + values[i++] = Int64GetDatum(wentry->xact_abort); + values[i++] = Int64GetDatum(wentry->xact_abort_bytes); + + /* last_error_relid */ if (OidIsValid(wentry->relid)) values[i++] = ObjectIdGetDatum(wentry->relid); else nulls[i++] = true; - /* command */ + /* last_error_command */ if (wentry->command != 0) values[i++] = CStringGetTextDatum(logicalrep_message_type(wentry->command)); else nulls[i++] = true; - /* xid */ + /* last_error_xid */ if (TransactionIdIsValid(wentry->xid)) values[i++] = TransactionIdGetDatum(wentry->xid); else nulls[i++] = true; - /* count */ + /* last_error_count */ values[i++] = Int64GetDatum(wentry->count); - /* error_message */ + /* last_error_message */ values[i++] = CStringGetTextDatum(wentry->message); /* last_error_time */ diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index 4543a00..98bf9c1 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -5385,13 +5385,13 @@ proargmodes => '{i,o,o,o,o,o,o,o,o,o,o}', proargnames => '{slot_name,slot_name,spill_txns,spill_count,spill_bytes,stream_txns,stream_count,stream_bytes,total_txns,total_bytes,stats_reset}', prosrc => 'pg_stat_get_replication_slot' }, -{ oid => '8523', descr => 'statistics: information about subscription error', +{ oid => '8523', descr => 'statistics: information about subscription', proname => 'pg_stat_get_subscription_worker', prorows => '1', proisstrict => 'f', proretset => 't', provolatile => 's', proparallel => 'r', prorettype => 'record', proargtypes => 'oid oid', - proallargtypes => '{oid,oid,oid,oid,oid,text,xid,int8,text,timestamptz,timestamptz}', - proargmodes => '{i,i,o,o,o,o,o,o,o,o,o}', - proargnames => '{subid,subrelid,subid,subrelid,relid,command,xid,count,error_message,last_error_time,stats_reset}', + proallargtypes => '{oid,oid,oid,oid,int8,int8,int8,int8,int8,int8,oid,text,xid,int8,text,timestamptz,timestamptz}', + proargmodes => '{i,i,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o}', + proargnames => '{subid,subrelid,subid,subrelid,xact_commit,xact_commit_bytes,xact_error,xact_error_bytes,xact_abort,xact_abort_bytes,last_error_relid,last_error_command,last_error_xid,last_error_count,last_error_message,last_error_time,stats_reset}', prosrc => 'pg_stat_get_subscription_worker' }, { oid => '6118', descr => 'statistics: information about subscription', proname => 'pg_stat_get_subscription', prorows => '10', proisstrict => 'f', diff --git a/src/include/executor/execPartition.h b/src/include/executor/execPartition.h index 694e38b..773e46c 100644 --- a/src/include/executor/execPartition.h +++ b/src/include/executor/execPartition.h @@ -110,6 +110,7 @@ typedef struct PartitionPruneState PartitionPruningData *partprunedata[FLEXIBLE_ARRAY_MEMBER]; } PartitionPruneState; +extern size_t PartitionTupleRoutingSize(void); extern PartitionTupleRouting *ExecSetupPartitionTupleRouting(EState *estate, Relation rel); extern ResultRelInfo *ExecFindPartition(ModifyTableState *mtstate, diff --git a/src/include/pgstat.h b/src/include/pgstat.h index 59e3763..33bc169 100644 --- a/src/include/pgstat.h +++ b/src/include/pgstat.h @@ -88,6 +88,8 @@ typedef enum StatMsgType PGSTAT_MTYPE_SUBSCRIPTIONPURGE, PGSTAT_MTYPE_SUBWORKERERROR, PGSTAT_MTYPE_SUBWORKERPURGE, + PGSTAT_MTYPE_SUBWORKERXACTEND, + PGSTAT_MTYPE_SUBWORKERTWOPHASEXACT, } StatMsgType; /* ---------- @@ -591,6 +593,54 @@ typedef struct PgStat_MsgSubWorkerPurge } PgStat_MsgSubWorkerPurge; /* ---------- + * PgStat_MsgSubscriptionXactEnd Sent by the apply worker or the table sync worker + * to report successful transaction ends. + * ---------- + */ +typedef struct PgStat_MsgSubWorkerXactEnd +{ + PgStat_MsgHdr m_hdr; + + /* determine the worker entry */ + Oid m_subid; + Oid m_subrelid; + + /* + * distinguish between transaction commits and streaming transaction aborts + * that are handled without error. + */ + LogicalRepMsgType m_command; + + /* memory consumption used by transaction */ + PgStat_Counter m_xact_bytes; + +} PgStat_MsgSubWorkerXactEnd; + +/* ---------- + * PgStat_MsgSubWorkerTwophaseXact Sent by the apply worker to make size of prepared + * txn persistent over the server restart and make it + * visible after commit prepare or rollback prepared. + * This is separated from PgStat_MsgSubWorkerXactEnd + * so that we can reduce message size of gid for other + * operations (e.g. normal COMMIT) that should happen more + * frequently than prepare operation usually. + * ---------- + */ +typedef struct PgStat_MsgSubWorkerTwophaseXact +{ + PgStat_MsgHdr m_hdr; + + /* determine the subscription */ + Oid m_subid; + + LogicalRepMsgType m_command; + char m_gid[GIDSIZE]; + int gid_len; + PgStat_Counter m_xact_bytes; + +} PgStat_MsgSubWorkerTwophaseXact; + +/* ---------- * PgStat_MsgSubWorkerError Sent by the apply worker or the table sync worker to * report the error occurred during logical replication. * ---------- @@ -609,6 +659,12 @@ typedef struct PgStat_MsgSubWorkerError Oid m_subrelid; /* + * Transaction stats of subscription needs to be updated when an + * error occurs. + */ + PgStat_Counter m_xact_error_bytes; + + /* * Oids of the database and the table that the reporter was actually * processing. m_relid can be InvalidOid if an error occurred during * worker applying a non-data-modification message such as RELATION. @@ -803,6 +859,8 @@ typedef union PgStat_Msg PgStat_MsgSubscriptionPurge msg_subscriptionpurge; PgStat_MsgSubWorkerError msg_subworkererror; PgStat_MsgSubWorkerPurge msg_subworkerpurge; + PgStat_MsgSubWorkerXactEnd msg_subworkerxactend; + PgStat_MsgSubWorkerTwophaseXact msg_subworkertwophasexact; } PgStat_Msg; @@ -1035,6 +1093,16 @@ typedef struct PgStat_StatSubWorkerEntry PgStat_StatSubWorkerKey key; /* hash key (must be first) */ /* + * Transaction statistics of subscription worker + */ + PgStat_Counter xact_commit; + PgStat_Counter xact_commit_bytes; + PgStat_Counter xact_error; + PgStat_Counter xact_error_bytes; + PgStat_Counter xact_abort; + PgStat_Counter xact_abort_bytes; + + /* * Subscription worker error statistics representing an error that * occurred during application of logical replication or the initial table * synchronization. @@ -1049,6 +1117,22 @@ typedef struct PgStat_StatSubWorkerEntry TimestampTz stat_reset_timestamp; } PgStat_StatSubWorkerEntry; +/* prepared transaction */ +typedef struct PgStat_StatSubWorkerPreparedXact +{ + Oid subid; + char gid[GIDSIZE]; +} PgStat_StatSubWorkerPreparedXact; + +typedef struct PgStat_StatSubWorkerPreparedXactSize +{ + PgStat_StatSubWorkerPreparedXact key; /* hash key */ + + Oid subid; + char gid[GIDSIZE]; + PgStat_Counter xact_size; +} PgStat_StatSubWorkerPreparedXactSize; + /* * Working state needed to accumulate per-function-call timing statistics. */ @@ -1158,6 +1242,13 @@ extern void pgstat_report_checksum_failure(void); extern void pgstat_report_replslot(const PgStat_StatReplSlotEntry *repSlotStat); extern void pgstat_report_replslot_create(const char *slotname); extern void pgstat_report_replslot_drop(const char *slotname); +extern void pgstat_report_subworker_xact_end(Oid subid, Oid subrel, + LogicalRepMsgType command, PgStat_Counter xact_size); +extern void pgstat_report_subworker_twophase_xact(Oid subid, LogicalRepMsgType command, + PgStat_Counter xact_size, + LogicalRepPreparedTxnData *prepared_data, + LogicalRepCommitPreparedTxnData *commit_data, + LogicalRepRollbackPreparedTxnData *rollback_data); extern void pgstat_report_subworker_error(Oid subid, Oid subrelid, Oid relid, LogicalRepMsgType command, TransactionId xid, const char *errmsg); diff --git a/src/include/replication/logicalworker.h b/src/include/replication/logicalworker.h index 2ad61a0..9a8447b 100644 --- a/src/include/replication/logicalworker.h +++ b/src/include/replication/logicalworker.h @@ -16,4 +16,9 @@ extern void ApplyWorkerMain(Datum main_arg); extern bool IsLogicalWorker(void); +/* for transaction stats */ +extern int64 get_apply_error_context_xact_size(void); +extern void add_apply_error_context_xact_size(int64 size); +extern void reset_apply_error_context_xact_size(void); + #endif /* LOGICALWORKER_H */ diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out index c6b83ce..eac859f 100644 --- a/src/test/regress/expected/rules.out +++ b/src/test/regress/expected/rules.out @@ -2097,11 +2097,17 @@ pg_stat_subscription| SELECT su.oid AS subid, pg_stat_subscription_workers| SELECT e.subid, s.subname, e.subrelid, - e.relid, - e.command, - e.xid, - e.count, - e.error_message, + e.xact_commit, + e.xact_commit_bytes, + e.xact_error, + e.xact_error_bytes, + e.xact_abort, + e.xact_abort_bytes, + e.last_error_relid, + e.last_error_command, + e.last_error_xid, + e.last_error_count, + e.last_error_message, e.last_error_time, e.stats_reset FROM ( SELECT pg_subscription.oid AS subid, @@ -2112,7 +2118,7 @@ pg_stat_subscription_workers| SELECT e.subid, pg_subscription_rel.srrelid AS relid FROM pg_subscription_rel WHERE (pg_subscription_rel.srsubstate <> 'r'::"char")) sr, - (LATERAL pg_stat_get_subscription_worker(sr.subid, sr.relid) e(subid, subrelid, relid, command, xid, count, error_message, last_error_time, stats_reset) + (LATERAL pg_stat_get_subscription_worker(sr.subid, sr.relid) e(subid, subrelid, xact_commit, xact_commit_bytes, xact_error, xact_error_bytes, xact_abort, xact_abort_bytes, last_error_relid, last_error_command, last_error_xid, last_error_count, last_error_message, last_error_time, stats_reset) JOIN pg_subscription s ON ((e.subid = s.oid))); pg_stat_sys_indexes| SELECT pg_stat_all_indexes.relid, pg_stat_all_indexes.indexrelid, diff --git a/src/test/subscription/t/025_error_report.pl b/src/test/subscription/t/025_error_report.pl index 77d22ee..8ab403f 100644 --- a/src/test/subscription/t/025_error_report.pl +++ b/src/test/subscription/t/025_error_report.pl @@ -6,8 +6,8 @@ use strict; use warnings; -use PostgresNode; -use TestLib; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; use Test::More tests => 14; # Test if the error reported on pg_subscription_workers view is expected. @@ -17,8 +17,8 @@ sub test_subscription_error my $check_sql = qq[ SELECT count(1) > 0 FROM pg_stat_subscription_workers -WHERE relid = '$relname'::regclass]; - $check_sql .= " AND xid = '$xid'::xid;" if $xid ne ''; +WHERE last_error_relid = '$relname'::regclass]; + $check_sql .= " AND last_error_xid = '$xid'::xid;" if $xid ne ''; # Wait for the error statistics to be updated. $node->poll_query_until( @@ -28,9 +28,9 @@ WHERE relid = '$relname'::regclass]; my $result = $node->safe_psql( 'postgres', qq[ -SELECT subname, command, relid::regclass, count > 0 +SELECT subname, last_error_command, last_error_relid::regclass, last_error_count > 0 FROM pg_stat_subscription_workers -WHERE relid = '$relname'::regclass; +WHERE last_error_relid = '$relname'::regclass; ]); is($result, $expected_error, $msg); } @@ -46,7 +46,7 @@ sub test_skip_subscription_error # Get XID of the failed transaction. my $skipxid = $node->safe_psql( 'postgres', - "SELECT xid FROM pg_stat_subscription_workers WHERE relid = '$relname'::regclass"); + "SELECT last_error_xid FROM pg_stat_subscription_workers WHERE last_error_relid = '$relname'::regclass"); is($skipxid, $xid, "remote xid and skip_xid are equal"); $node->safe_psql('postgres', @@ -65,7 +65,7 @@ WHERE subname = '$subname' } # Create publisher node. -my $node_publisher = PostgresNode->new('publisher'); +my $node_publisher = PostgreSQL::Test::Cluster->new('publisher'); $node_publisher->init(allows_streaming => 'logical'); $node_publisher->append_conf('postgresql.conf', qq[ @@ -75,7 +75,7 @@ logical_decoding_work_mem = 64kB $node_publisher->start; # Create subscriber node. -my $node_subscriber = PostgresNode->new('subscriber'); +my $node_subscriber = PostgreSQL::Test::Cluster->new('subscriber'); $node_subscriber->init(allows_streaming => 'logical'); # The subscriber will enter an infinite error loop, so we don't want diff --git a/src/test/subscription/t/026_worker_xact_stats.pl b/src/test/subscription/t/026_worker_xact_stats.pl new file mode 100644 index 0000000..01b7e8a --- /dev/null +++ b/src/test/subscription/t/026_worker_xact_stats.pl @@ -0,0 +1,133 @@ + +# Copyright (c) 2021, PostgreSQL Global Development Group + +# Tests for subscription worker statistics during apply. +use strict; +use warnings; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More tests => 3; + +# Create publisher node +my $node_publisher = PostgreSQL::Test::Cluster->new('publisher'); +$node_publisher->init(allows_streaming => 'logical'); +$node_publisher->append_conf('postgresql.conf', qq[ +logical_decoding_work_mem = 64kB +max_prepared_transactions = 10 +max_wal_senders = 10 +wal_sender_timeout = 0 +]); +$node_publisher->start; + +# Create subscriber node +my $node_subscriber = PostgreSQL::Test::Cluster->new('subscriber'); +$node_subscriber->init(allows_streaming => 'logical'); +$node_subscriber->append_conf('postgresql.conf', qq[ +max_prepared_transactions = 10 +]); +$node_subscriber->start; + +# Create some preexisting content on publisher +$node_publisher->safe_psql('postgres', + "CREATE TABLE test_tab (a int)"); + +# Setup structure on subscriber +$node_subscriber->safe_psql('postgres', + "CREATE TABLE test_tab (a int primary key);"); + +# Setup logical replication +my $publisher_connstr = $node_publisher->connstr . ' dbname=postgres'; +$node_publisher->safe_psql('postgres', + "CREATE PUBLICATION tap_pub FOR TABLE test_tab"); + +my $appname = 'tap_sub'; +$node_subscriber->safe_psql('postgres', +"CREATE SUBSCRIPTION tap_sub CONNECTION '$publisher_connstr application_name=$appname' PUBLICATION tap_pub WITH (streaming = on, two_phase = on, copy_data = off);" +); + +$node_publisher->wait_for_catchup($appname); + +# COMMIT; +$node_publisher->safe_psql( + 'postgres', + "BEGIN; INSERT INTO test_tab VALUES (1); COMMIT;"); + +$node_subscriber->poll_query_until('postgres', +"SELECT count(1) = 1 FROM pg_stat_subscription_workers where xact_commit = 1;") + or die "didn't get updates of xact stats by commit"; + +# Now, stats collector make the bytes updated also. +my $result = $node_subscriber->safe_psql('postgres', +"SELECT xact_commit_bytes > 0 FROM pg_stat_subscription_workers;"); +is($result, 't', 'got consumed bytes'); + +# STREAM COMMIT +$node_publisher->safe_psql( + 'postgres', + "BEGIN; INSERT INTO test_tab VALUES(generate_series(1001, 2000)); COMMIT;"); +$result = $node_publisher->safe_psql('postgres', +"SELECT stream_count > 0 FROM pg_stat_replication_slots;"); +is($result, 't', "confirm the streaming has happened."); +$node_subscriber->poll_query_until('postgres', +"SELECT count(1) = 1 FROM pg_stat_subscription_workers where xact_commit = 2;") + or die "didn't get updates of xact stats by stream commit"; + +# STREAM PREPARE & COMMIT PREPARED that restores the xact size before the shutdown. +$node_publisher->safe_psql( + 'postgres', q[ +BEGIN; +INSERT INTO test_tab VALUES(generate_series(2001, 3000)); +PREPARE TRANSACTION 'gid1'; +]); + +# fetch the last value of xact_commit_bytes to check if +# this column is increased after the server restart and commit prepared. +my $tmp = $node_subscriber->safe_psql('postgres', +"SELECT xact_commit_bytes FROM pg_stat_subscription_workers;"); +$node_subscriber->restart; +$result = $node_publisher->safe_psql('postgres', "COMMIT PREPARED 'gid1'"); + +# stream prepare didn't increment xact_commit but commit prepared does. +$node_subscriber->poll_query_until('postgres', +"SELECT count(1) = 1 FROM pg_stat_subscription_workers where xact_commit = 3;") + or die "didn't get updates of xact stats by stream prepare and commit prepared"; + +$result = $node_subscriber->safe_psql('postgres', +"SELECT xact_commit_bytes > $tmp FROM pg_stat_subscription_workers;"); +is($result, 't', "after commit prepared, the xact size is restored"); + +# STREAM ABORT +$node_publisher->safe_psql('postgres', q[ +BEGIN; +INSERT INTO test_tab VALUES(generate_series(3001, 4000)); +ROLLBACK; +]); + +$node_subscriber->poll_query_until('postgres', +"SELECT count(1) = 1 FROM pg_stat_subscription_workers where xact_abort = 1;") + or die "didn't get updates of xact stats by stream abort"; + +# ROLLBACK PREPARED +$node_publisher->safe_psql('postgres', q[ +BEGIN; +INSERT INTO test_tab VALUES (2); +PREPARE TRANSACTION 'gid2'; +]); +$node_publisher->safe_psql('postgres', "ROLLBACK PREPARED 'gid2'"); +$node_subscriber->poll_query_until('postgres', +"SELECT count(1) = 1 FROM pg_stat_subscription_workers where xact_abort = 2;") + or die "didn't get updates of xact stats by rollback prepared"; + +# error stats (by duplication error) +$node_publisher->safe_psql('postgres', q[ +BEGIN; +INSERT INTO test_tab VALUES (1); -- already inserted this value +COMMIT; +]); + +$node_subscriber->poll_query_until('postgres', +"SELECT count(1) = 1 FROM pg_stat_subscription_workers where xact_error > 0;") + or die "didn't get updates of xact stats by error"; + +$node_subscriber->stop('fast'); +$node_publisher->stop('fast'); ^ permalink raw reply [nested|flat] 113+ messages in thread
* RE: Failed transaction statistics to measure the logical replication progress @ 2021-11-01 13:18 [email protected] <[email protected]> parent: [email protected] <[email protected]> 0 siblings, 2 replies; 113+ messages in thread From: [email protected] @ 2021-11-01 13:18 UTC (permalink / raw) To: 'Masahiko Sawada' <[email protected]>; Amit Kapila <[email protected]>; +Cc: [email protected] <[email protected]>; [email protected] <[email protected]> On Thursday, October 28, 2021 11:19 PM I wrote: > I've created a new patch that extends pg_stat_subscription_workers to include > other transaction statistics. > > Note that this patch depends on v18 patch-set in [1]... Rebased based on the v19 in [1]. Also, fixed documentation a little and made tests tidy. FYI, the newly included TAP test(027_worker_xact_stats.pl) is stable because I checked that 100 times of its execution in a tight loop all passed. [1] - https://www.postgresql.org/message-id/CAD21AoDY-9_x819F_m1_wfCVXXFJrGiSmR2MfC9Nw4nW8Om0qA%40mail.gma... Best Regards, Takamichi Osumi Attachments: [application/octet-stream] extend_xact_stats_of_subscription_worker_v7.patch (55.7K, ../../OSBPR01MB48880D8D2966FE8711F5BBB0ED8A9@OSBPR01MB4888.jpnprd01.prod.outlook.com/2-extend_xact_stats_of_subscription_worker_v7.patch) download | inline diff: diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml index 094c723..0006c64 100644 --- a/doc/src/sgml/monitoring.sgml +++ b/doc/src/sgml/monitoring.sgml @@ -629,7 +629,7 @@ postgres 27093 0.0 0.0 30096 2752 ? Ss 11:34 0:00 postgres: ser <row> <entry><structname>pg_stat_subscription_workers</structname><indexterm><primary>pg_stat_subscription_workers</primary></indexterm></entry> - <entry>At least one row per subscription, showing about errors that + <entry>At least one row per subscription, showing about transaction statistics and error summary that occurred on subscription. See <link linkend="monitoring-pg-stat-subscription-workers"> <structname>pg_stat_subscription_workers</structname></link> for details. @@ -3052,9 +3052,9 @@ SELECT pid, wait_event_type, wait_event FROM pg_stat_activity WHERE wait_event i <para> The <structname>pg_stat_subscription_workers</structname> view will contain - one row per subscription error reported by workers applying logical - replication changes and workers handling the initial data copy of the - subscribed tables. + one row per subscription for transaction statistics and summary of the last + error reported by workers applying logical replication changes and workers + handling the initial data copy of the subscribed tables. </para> <table id="pg-stat-subscription-workers" xreflabel="pg_stat_subscription_workers"> @@ -3102,20 +3102,86 @@ SELECT pid, wait_event_type, wait_event FROM pg_stat_activity WHERE wait_event i <row> <entry role="catalog_table_entry"><para role="column_definition"> - <structfield>relid</structfield> <type>oid</type> + <structfield>xact_commit</structfield> <type>bigint</type> + </para> + <para> + Number of transactions successfully applied in this subscription. + COMMIT, COMMIT of streaming transaction and COMMIT PREPARED increments + this counter. + </para></entry> + </row> + + <row> + <entry role="catalog_table_entry"><para role="column_definition"> + <structfield>xact_commit_bytes</structfield> <type>bigint</type> + </para> + <para> + Amount of transactions data successfully applied in this subscription. + Consumed memory for xact_commit is displayed. + </para></entry> + </row> + + <row> + <entry role="catalog_table_entry"><para role="column_definition"> + <structfield>xact_error</structfield> <type>bigint</type> + </para> + <para> + Number of transactions failed to be applied and caught by table + sync worker or main apply worker in this subscription. + </para></entry> + </row> + + <row> + <entry role="catalog_table_entry"><para role="column_definition"> + <structfield>xact_error_bytes</structfield> <type>bigint</type> + </para> + <para> + Amount of transactions data unsuccessfully applied in this subscription. + Consumed memory that past failed transaction used is displayed. + </para></entry> + </row> + + <row> + <entry role="catalog_table_entry"><para role="column_definition"> + <structfield>xact_abort</structfield> <type>bigint</type> + </para> + <para> + Number of transactions aborted in this subscription. + Rollback of the prepared transaction and abort of streaming transaction + increments this counter. + </para></entry> + </row> + + <row> + <entry role="catalog_table_entry"><para role="column_definition"> + <structfield>xact_abort_bytes</structfield> <type>bigint</type> + </para> + <para> + Amount of transactions data aborted in this subscription. + Increase <literal>logical_decoding_work_mem</literal> on the publisher + so that it exceeds the size of whole streamed transaction + to suppress unnecessary consumed network bandwidth in addition to change + in memory of the subscriber, if unexpected amount of streamed transactions + are aborted. + </para></entry> + </row> + + <row> + <entry role="catalog_table_entry"><para role="column_definition"> + <structfield>last_error_relid</structfield> <type>oid</type> </para> <para> OID of the relation that the worker was processing when the - error occurred + last error occurred </para></entry> </row> <row> <entry role="catalog_table_entry"><para role="column_definition"> - <structfield>command</structfield> <type>text</type> + <structfield>last_error_command</structfield> <type>text</type> </para> <para> - Name of command being applied when the error occurred. This field + Name of last command being applied when the error occurred. This field is always NULL if the error was reported during the initial data copy. </para></entry> @@ -3123,10 +3189,10 @@ SELECT pid, wait_event_type, wait_event FROM pg_stat_activity WHERE wait_event i <row> <entry role="catalog_table_entry"><para role="column_definition"> - <structfield>xid</structfield> <type>xid</type> + <structfield>last_error_xid</structfield> <type>xid</type> </para> <para> - Transaction ID of the publisher node being applied when the error + Transaction ID of the publisher node being applied when the last error occurred. This field is always NULL if the error was reported during the initial data copy. </para></entry> @@ -3134,19 +3200,19 @@ SELECT pid, wait_event_type, wait_event FROM pg_stat_activity WHERE wait_event i <row> <entry role="catalog_table_entry"><para role="column_definition"> - <structfield>error_count</structfield> <type>uint8</type> + <structfield>last_error_count</structfield> <type>uint8</type> </para> <para> - Number of consecutive times the error occurred + Number of consecutive times the last error occurred </para></entry> </row> <row> <entry role="catalog_table_entry"><para role="column_definition"> - <structfield>error_message</structfield> <type>text</type> + <structfield>last_error_message</structfield> <type>text</type> </para> <para> - The error message + The last error message </para></entry> </row> diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index a2ee00c..312f145 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1267,11 +1267,17 @@ CREATE VIEW pg_stat_subscription_workers AS w.subid, s.subname, w.subrelid, - w.relid, - w.command, - w.xid, - w.error_count, - w.error_message, + w.xact_commit, + w.xact_commit_bytes, + w.xact_error, + w.xact_error_bytes, + w.xact_abort, + w.xact_abort_bytes, + w.last_error_relid, + w.last_error_command, + w.last_error_xid, + w.last_error_count, + w.last_error_message, w.last_error_time, w.stats_reset FROM (SELECT diff --git a/src/backend/executor/execPartition.c b/src/backend/executor/execPartition.c index 5c723bc..20184c6 100644 --- a/src/backend/executor/execPartition.c +++ b/src/backend/executor/execPartition.c @@ -192,6 +192,16 @@ static void find_matching_subplans_recurse(PartitionPruningData *prunedata, bool initial_prune, Bitmapset **validsubplans); +/* + * PartitionTupleRoutingSize - exported to calculate total data size + * of logical replication mesage apply, because this is one of the + * ApplyExecutionData struct members. + */ +size_t +PartitionTupleRoutingSize(void) +{ + return sizeof(PartitionTupleRouting); +} /* * ExecSetupPartitionTupleRouting - sets up information needed during diff --git a/src/backend/postmaster/pgstat.c b/src/backend/postmaster/pgstat.c index b7883ec..d71049e 100644 --- a/src/backend/postmaster/pgstat.c +++ b/src/backend/postmaster/pgstat.c @@ -54,6 +54,7 @@ #include "postmaster/fork_process.h" #include "postmaster/interrupt.h" #include "postmaster/postmaster.h" +#include "replication/logicalworker.h" #include "replication/slot.h" #include "replication/walsender.h" #include "storage/backendid.h" @@ -288,6 +289,15 @@ static HTAB *replSlotStatHash = NULL; static HTAB *subWorkerStatHash = NULL; /* + * Stats of prepared transactions should be displayed + * at either commit prepared or rollback prepared time, even when it's + * after the server restart. We have the apply worker send those statistics + * to the stats collector at prepare time and the startup process restore + * those at restart if necessary. + */ +static HTAB *subWorkerPreparedXactSizeHash = NULL; + +/* * List of OIDs of databases we need to write out. If an entry is InvalidOid, * it means to write only the shared-catalog stats ("DB 0"); otherwise, we * will write both that DB's data and the shared stats. @@ -338,6 +348,9 @@ static void pgstat_reset_replslot(PgStat_StatReplSlotEntry *slotstats, Timestamp static PgStat_StatSubWorkerEntry *pgstat_get_subworker_entry(Oid subid, Oid subrelid, bool create); +static PgStat_StatSubWorkerPreparedXactSize *pgstat_get_subworker_prepared_txn(Oid subid, + char *gid, bool create); + static void pgstat_reset_subworker_entry(PgStat_StatSubWorkerEntry *wentry, TimestampTz ts); static void pgstat_vacuum_subworker_stats(void); static void pgstat_send_subscription_purge(PgStat_MsgSubscriptionPurge *msg); @@ -388,6 +401,9 @@ static void pgstat_recv_tempfile(PgStat_MsgTempFile *msg, int len); static void pgstat_recv_subscription_purge(PgStat_MsgSubscriptionPurge *msg, int len); static void pgstat_recv_subworker_error(PgStat_MsgSubWorkerError *msg, int len); static void pgstat_recv_subworker_purge(PgStat_MsgSubWorkerPurge *msg, int len); +static void pgstat_recv_subworker_xact_end(PgStat_MsgSubWorkerXactEnd *msg, int len); +static void pgstat_recv_subworker_twophase_xact(PgStat_MsgSubWorkerTwophaseXact *msg, int len); + /* ------------------------------------------------------------ * Public functions called from postmaster follow @@ -2111,6 +2127,72 @@ pgstat_report_replslot_drop(const char *slotname) } /* ---------- + * pgstat_report_subworker_xact_end() - + * + * Tell the collector that worker transaction has finished without problem. + * ---------- + */ +void +pgstat_report_subworker_xact_end(Oid subid, Oid subrel, + LogicalRepMsgType command, PgStat_Counter xact_size) +{ + PgStat_MsgSubWorkerXactEnd msg; + + Assert(command == 0 /* table sync worker */ || + command == LOGICAL_REP_MSG_COMMIT || + command == LOGICAL_REP_MSG_STREAM_ABORT || + command == LOGICAL_REP_MSG_STREAM_COMMIT); + + pgstat_setheader(&msg.m_hdr, PGSTAT_MTYPE_SUBWORKERXACTEND); + msg.m_subid = subid; + msg.m_subrelid = subrel; + msg.m_command = command; + msg.m_xact_bytes = xact_size; + pgstat_send(&msg, sizeof(PgStat_MsgSubWorkerXactEnd)); + + reset_apply_error_context_xact_size(); +} + +/* ---------- + * pgstat_report_subworker_twophase_xact() - + * + * Tell the collector that worker transaction has done 2PC related operation. + * ---------- + */ +void +pgstat_report_subworker_twophase_xact(Oid subid, LogicalRepMsgType command, + PgStat_Counter xact_size, + LogicalRepPreparedTxnData *prepare_data, + LogicalRepCommitPreparedTxnData *commit_data, + LogicalRepRollbackPreparedTxnData *rollback_data) +{ + PgStat_MsgSubWorkerTwophaseXact msg; + + Assert(command == LOGICAL_REP_MSG_PREPARE || + command == LOGICAL_REP_MSG_STREAM_PREPARE || + command == LOGICAL_REP_MSG_COMMIT_PREPARED || + command == LOGICAL_REP_MSG_ROLLBACK_PREPARED); + + /* setup the message */ + pgstat_setheader(&msg.m_hdr, PGSTAT_MTYPE_SUBWORKERTWOPHASEXACT); + msg.m_subid = subid; + msg.m_command = command; + + /* get the gid for this two phase operation */ + if (command == LOGICAL_REP_MSG_PREPARE || + command == LOGICAL_REP_MSG_STREAM_PREPARE) + memcpy(msg.m_gid, prepare_data->gid, GIDSIZE); + else if (command == LOGICAL_REP_MSG_COMMIT_PREPARED) + memcpy(msg.m_gid, commit_data->gid, GIDSIZE); + else /* rollback prepared */ + memcpy(msg.m_gid, rollback_data->gid, GIDSIZE); + + msg.m_xact_bytes = xact_size; + pgstat_send(&msg, sizeof(PgStat_MsgSubWorkerTwophaseXact)); + reset_apply_error_context_xact_size(); +} + +/* ---------- * pgstat_report_subworker_error() - * * Tell the collector about the subscription worker error. @@ -2130,6 +2212,7 @@ pgstat_report_subworker_error(Oid subid, Oid subrelid, Oid relid, pgstat_setheader(&msg.m_hdr, PGSTAT_MTYPE_SUBWORKERERROR); msg.m_subid = subid; msg.m_subrelid = subrelid; + msg.m_xact_error_bytes = get_apply_error_context_xact_size(); msg.m_dbid = MyDatabaseId; msg.m_relid = relid; msg.m_command = command; @@ -2138,6 +2221,8 @@ pgstat_report_subworker_error(Oid subid, Oid subrelid, Oid relid, strlcpy(msg.m_message, errmsg, PGSTAT_SUBWORKERERROR_MSGLEN); pgstat_send(&msg, len); + + reset_apply_error_context_xact_size(); } /* ---------- @@ -3906,6 +3991,14 @@ PgstatCollectorMain(int argc, char *argv[]) pgstat_recv_subworker_purge(&msg.msg_subworkerpurge, len); break; + case PGSTAT_MTYPE_SUBWORKERXACTEND: + pgstat_recv_subworker_xact_end(&msg.msg_subworkerxactend, len); + break; + + case PGSTAT_MTYPE_SUBWORKERTWOPHASEXACT: + pgstat_recv_subworker_twophase_xact(&msg.msg_subworkertwophasexact, len); + break; + default: break; } @@ -4223,6 +4316,22 @@ pgstat_write_statsfiles(bool permanent, bool allDbs) } /* + * Write subscription worker's prepared transaction struct + */ + if (subWorkerPreparedXactSizeHash) + { + PgStat_StatSubWorkerPreparedXactSize *prepared_size; + + hash_seq_init(&hstat, subWorkerPreparedXactSizeHash); + while((prepared_size = (PgStat_StatSubWorkerPreparedXactSize *) hash_seq_search(&hstat)) != NULL) + { + fputc('P', fpout); + rc = fwrite(prepared_size, sizeof(PgStat_StatSubWorkerPreparedXactSize), 1, fpout); + (void) rc; /* we'll check for error with ferror */ + } + } + + /* * No more output to be done. Close the temp file and replace the old * pgstat.stat with it. The ferror() check replaces testing for error * after each individual fputc or fwrite above. @@ -4725,6 +4834,40 @@ pgstat_read_statsfiles(Oid onlydb, bool permanent, bool deep) break; } + case 'P': + { + PgStat_StatSubWorkerPreparedXactSize buff; + PgStat_StatSubWorkerPreparedXactSize *prepared_xact_size; + + if (fread(&buff, 1, sizeof(PgStat_StatSubWorkerPreparedXactSize), + fpin) != sizeof(PgStat_StatSubWorkerPreparedXactSize)) + { + ereport(pgStatRunningInCollector ? LOG : WARNING, + (errmsg("corrupted statistics file \"%s\"", + statfile))); + goto done; + } + + if (subWorkerPreparedXactSizeHash == NULL) + { + HASHCTL hash_ctl; + + hash_ctl.keysize = sizeof(PgStat_StatSubWorkerPreparedXact); + hash_ctl.entrysize = sizeof(PgStat_StatSubWorkerPreparedXactSize); + hash_ctl.hcxt = pgStatLocalContext; + subWorkerPreparedXactSizeHash = hash_create("Subscription worker stats of prepared txn", + PGSTAT_SUBWORKER_HASH_SIZE, + &hash_ctl, + HASH_ELEM | HASH_STRINGS | HASH_CONTEXT); + } + + prepared_xact_size = + (PgStat_StatSubWorkerPreparedXactSize *) hash_search(subWorkerPreparedXactSizeHash, + (void *) &buff.key, + HASH_ENTER, NULL); + memcpy(prepared_xact_size, &buff, sizeof(PgStat_StatSubWorkerPreparedXactSize)); + break; + } case 'E': goto done; @@ -5290,6 +5433,7 @@ pgstat_clear_snapshot(void) pgStatDBHash = NULL; replSlotStatHash = NULL; subWorkerStatHash = NULL; + subWorkerPreparedXactSizeHash = NULL; /* * Historically the backend_status.c facilities lived in this file, and @@ -6287,6 +6431,105 @@ pgstat_recv_subscription_purge(PgStat_MsgSubscriptionPurge *msg, int len) } /* ---------- + * pgstat_recv_subworker_xact_end() - + * + * Process a SUBWORKERXACTEND message. + * ---------- + */ +static void +pgstat_recv_subworker_xact_end(PgStat_MsgSubWorkerXactEnd *msg, int len) +{ + PgStat_StatSubWorkerEntry *wentry; + + wentry = pgstat_get_subworker_entry(msg->m_subid, msg->m_subrelid, true); + Assert(wentry); + + /* table sync worker */ + if (msg->m_command == 0) + wentry->xact_commit++; + else + { + /* apply worker */ + switch(msg->m_command) + { + case LOGICAL_REP_MSG_COMMIT: + case LOGICAL_REP_MSG_STREAM_COMMIT: + wentry->xact_commit++; + wentry->xact_commit_bytes += msg->m_xact_bytes; + break; + case LOGICAL_REP_MSG_STREAM_ABORT: + wentry->xact_abort++; + wentry->xact_abort_bytes += msg->m_xact_bytes; + break; + default: + elog(ERROR, "unexpected logical message type as normal apply end"); + break; + } + } +} + +/* ---------- + * pgstat_recv_subworker_twophase_xact() - + * + * Process a SUBWORKERTWOPHASEXACT message. + * ---------- + */ +static void +pgstat_recv_subworker_twophase_xact(PgStat_MsgSubWorkerTwophaseXact *msg, int len) +{ + PgStat_StatSubWorkerPreparedXactSize *prepared_txn; + PgStat_StatSubWorkerEntry *wentry; + PgStat_StatSubWorkerPreparedXact key; + + prepared_txn = pgstat_get_subworker_prepared_txn(msg->m_subid, msg->m_gid, true); + Assert(prepared_txn); + switch(msg->m_command) + { + case LOGICAL_REP_MSG_PREPARE: + case LOGICAL_REP_MSG_STREAM_PREPARE: + /* + * Make each size of prepared transaction persistent + * so that we can update stats over the server restart + * and make prepared stats updated when commit prepared + * or rollback prepared arrives. + */ + prepared_txn->subid = msg->m_subid; + strlcpy(prepared_txn->gid, msg->m_gid, strlen(msg->m_gid) + 1); + prepared_txn->xact_size = msg->m_xact_bytes; + break; + + case LOGICAL_REP_MSG_COMMIT_PREPARED: + case LOGICAL_REP_MSG_ROLLBACK_PREPARED: + /* Update exported xact stats now */ + wentry = pgstat_get_subworker_entry(msg->m_subid, + InvalidOid /* apply worker */, + true); + Assert(wentry); + if (msg->m_command == LOGICAL_REP_MSG_COMMIT_PREPARED) + { + wentry->xact_commit++; + wentry->xact_commit_bytes += prepared_txn->xact_size; + } + else + { + wentry->xact_abort++; + wentry->xact_abort_bytes += prepared_txn->xact_size; + } + + /* Clean up this gid from transaction size hash */ + key.subid = prepared_txn->subid; + memcpy(key.gid, msg->m_gid, strlen(msg->m_gid)); + (void) hash_search(subWorkerPreparedXactSizeHash, + (void *) &key, HASH_REMOVE, NULL); + break; + + default: + elog(ERROR, "unexpected logical message type as prepare transaction"); + break; + } +} + +/* ---------- * pgstat_recv_subworker_error() - * * Process a SUBWORKERERROR message. @@ -6301,6 +6544,10 @@ pgstat_recv_subworker_error(PgStat_MsgSubWorkerError *msg, int len) wentry = pgstat_get_subworker_entry(msg->m_subid, msg->m_subrelid, true); Assert(wentry); + /* general transaction stats for error */ + wentry->xact_error++; + wentry->xact_error_bytes += msg->m_xact_error_bytes; + /* * Update only the counter and timestamp if we received the same error * again @@ -6520,6 +6767,51 @@ pgstat_get_subworker_entry(Oid subid, Oid subrelid, bool create) } /* ---------- + * pgstat_get_subworker_prepared_txn + * + * Return subscription worker entry with the given subscription OID and + * gid. + * ---------- + */ +static PgStat_StatSubWorkerPreparedXactSize* +pgstat_get_subworker_prepared_txn(Oid subid, char *gid, bool create) +{ + PgStat_StatSubWorkerPreparedXact key; + PgStat_StatSubWorkerPreparedXactSize *prepared_txn_size; + HASHACTION action; + bool found; + + if (subWorkerPreparedXactSizeHash == NULL) + { + HASHCTL hash_ctl; + + hash_ctl.keysize = sizeof(PgStat_StatSubWorkerPreparedXact); + hash_ctl.entrysize = sizeof(PgStat_StatSubWorkerPreparedXactSize); + hash_ctl.hcxt = pgStatLocalContext; + subWorkerPreparedXactSizeHash = hash_create("Subscription worker stats of prepared txn", + PGSTAT_SUBWORKER_HASH_SIZE, + &hash_ctl, + HASH_ELEM | HASH_STRINGS); + } + + key.subid = subid; + memcpy(key.gid, gid, strlen(gid)); + action = (create ? HASH_ENTER : HASH_FIND); + prepared_txn_size = (PgStat_StatSubWorkerPreparedXactSize *) hash_search(subWorkerPreparedXactSizeHash, + (void *) &key, + action, &found); + + if (create && !found) + { + prepared_txn_size->subid = 0; + prepared_txn_size->gid[0] = '\0'; + prepared_txn_size->xact_size = 0; + } + + return prepared_txn_size; +} + +/* ---------- * pgstat_reset_subworker_entry * * Reset the given subscription worker statistics. diff --git a/src/backend/replication/logical/tablesync.c b/src/backend/replication/logical/tablesync.c index 9b6d057..0d04ea5 100644 --- a/src/backend/replication/logical/tablesync.c +++ b/src/backend/replication/logical/tablesync.c @@ -1162,6 +1162,12 @@ copy_table_done: MyLogicalRepWorker->relstate_lsn = *origin_startpos; SpinLockRelease(&MyLogicalRepWorker->relmutex); + /* Report the success of table sync. */ + pgstat_report_subworker_xact_end(MyLogicalRepWorker->subid, + MyLogicalRepWorker->relid, + 0 /* no logical message type */, + 0 /* xact size */); + /* * Finally, wait until the main apply worker tells us to catch up and then * return to let LogicalRepApplyLoop do it. diff --git a/src/backend/replication/logical/worker.c b/src/backend/replication/logical/worker.c index 3a40684..1cbe921 100644 --- a/src/backend/replication/logical/worker.c +++ b/src/backend/replication/logical/worker.c @@ -221,6 +221,21 @@ typedef struct ApplyErrorCallbackArg LogicalRepMsgType command; /* 0 if invalid */ LogicalRepRelMapEntry *rel; + /* + * Store data size of this transaction. + * + * The byte size of transaction on the publisher is calculated + * by ReorderBufferChangeSize() based on the ReorderBufferChange + * structure. But on the subscriber, consumed resources are + * not same as the publisher's decoding processsing and required + * to be computed in different way. Therefore, the exact same byte + * size is not restored on the subscriber usually. + * + * Data size of streaming transactions is managed by streamingXactSize + * for flexible data blocks handling. + */ + PgStat_Counter bytes; + /* Remote node information */ int remote_attnum; /* -1 if invalid */ TransactionId remote_xid; @@ -231,11 +246,32 @@ static ApplyErrorCallbackArg apply_error_callback_arg = { .command = 0, .rel = NULL, + .bytes = 0, .remote_attnum = -1, .remote_xid = InvalidTransactionId, .ts = 0, }; +/* + * Two or more streaming transactions in parallel on the publisher + * generate unexpected order of partial txn data demarcated stream start + * and stream stop to the subscriber, since whenever its size of one of + * the txns reaches the publisher's logical_decoding_work_mem, + * the part (and in the end, last remaining changes) is streamed. + * This creates mixed blocks of streaming data while + * there's possibility some are successfully committed but others are + * not by stream abort. Therefore, to track correct byte size + * it's necessary to trace each streaming transaction by making pair + * of xid and transaction size. + */ +#define PARALLEL_STREAMING_XACTS 32 +typedef struct XactSizeEntry +{ + TransactionId key; + PgStat_Counter xact_size; +} XactSizeEntry; +static HTAB *streamingXactSize = NULL; + static MemoryContext ApplyMessageContext = NULL; MemoryContext ApplyContext = NULL; @@ -304,6 +340,8 @@ static void maybe_reread_subscription(void); /* prototype needed because of stream_commit */ static void apply_dispatch(StringInfo s); +void update_apply_change_size(LogicalRepMsgType action, void *data); + static void apply_handle_commit_internal(LogicalRepCommitData *commit_data); static void apply_handle_insert_internal(ApplyExecutionData *edata, ResultRelInfo *relinfo, @@ -818,6 +856,11 @@ apply_handle_commit(StringInfo s) /* Process any tables that are being synchronized in parallel. */ process_syncing_tables(commit_data.end_lsn); + pgstat_report_subworker_xact_end(MyLogicalRepWorker->subid, + InvalidOid, + LOGICAL_REP_MSG_COMMIT, + get_apply_error_context_xact_size()); + pgstat_report_activity(STATE_IDLE, NULL); reset_apply_error_context_info(); } @@ -922,6 +965,11 @@ apply_handle_prepare(StringInfo s) /* Process any tables that are being synchronized in parallel. */ process_syncing_tables(prepare_data.end_lsn); + pgstat_report_subworker_twophase_xact(MyLogicalRepWorker->subid, + LOGICAL_REP_MSG_PREPARE, + get_apply_error_context_xact_size(), + &prepare_data, NULL, NULL); + pgstat_report_activity(STATE_IDLE, NULL); reset_apply_error_context_info(); } @@ -963,6 +1011,13 @@ apply_handle_commit_prepared(StringInfo s) /* Process any tables that are being synchronized in parallel. */ process_syncing_tables(prepare_data.end_lsn); + update_apply_change_size(LOGICAL_REP_MSG_COMMIT_PREPARED, NULL); + + pgstat_report_subworker_twophase_xact(MyLogicalRepWorker->subid, + LOGICAL_REP_MSG_COMMIT_PREPARED, + get_apply_error_context_xact_size(), + NULL, &prepare_data, NULL); + pgstat_report_activity(STATE_IDLE, NULL); reset_apply_error_context_info(); } @@ -1014,6 +1069,12 @@ apply_handle_rollback_prepared(StringInfo s) /* Process any tables that are being synchronized in parallel. */ process_syncing_tables(rollback_data.rollback_end_lsn); + /* send rollback prepared message for this gid */ + pgstat_report_subworker_twophase_xact(MyLogicalRepWorker->subid, + LOGICAL_REP_MSG_ROLLBACK_PREPARED, + get_apply_error_context_xact_size(), + NULL, NULL, &rollback_data); + pgstat_report_activity(STATE_IDLE, NULL); reset_apply_error_context_info(); } @@ -1029,6 +1090,7 @@ static void apply_handle_stream_prepare(StringInfo s) { LogicalRepPreparedTxnData prepare_data; + XactSizeEntry *streamed_entry; if (in_streamed_transaction) ereport(ERROR, @@ -1066,6 +1128,19 @@ apply_handle_stream_prepare(StringInfo s) /* Process any tables that are being synchronized in parallel. */ process_syncing_tables(prepare_data.end_lsn); + /* + * Report the prepared streaming xact size to the stats collector + * in a prepared xact manner to make it survive over the restart. + */ + streamed_entry = hash_search(streamingXactSize, + (void *) &prepare_data.xid, + HASH_FIND, NULL); + Assert(streamed_entry); + pgstat_report_subworker_twophase_xact(MyLogicalRepWorker->subid, + LOGICAL_REP_MSG_STREAM_PREPARE, + streamed_entry->xact_size, + &prepare_data, NULL, NULL); + pgstat_report_activity(STATE_IDLE, NULL); reset_apply_error_context_info(); @@ -1143,6 +1218,8 @@ apply_handle_stream_start(StringInfo s) MyLogicalRepWorker->stream_fileset = palloc(sizeof(FileSet)); FileSetInit(MyLogicalRepWorker->stream_fileset); + update_apply_change_size(LOGICAL_REP_MSG_STREAM_START, NULL); + MemoryContextSwitchTo(oldctx); } @@ -1193,12 +1270,17 @@ apply_handle_stream_stop(StringInfo s) /* * Handle STREAM abort message. + * + * Currently, abort of streaming subtransaction does not affect + * size of streaming transaction resources because it have used the + * resources anyway. */ static void apply_handle_stream_abort(StringInfo s) { TransactionId xid; TransactionId subxid; + XactSizeEntry *streamed_entry; if (in_streamed_transaction) ereport(ERROR, @@ -1213,8 +1295,36 @@ apply_handle_stream_abort(StringInfo s) */ if (xid == subxid) { + bool found = false; set_apply_error_context_xact(xid, 0); stream_cleanup_files(MyLogicalRepWorker->subid, xid); + + /* + * We've completed to handle stream abort without issue, so + * get ready to report the transaction stats via normal + * termination route instead of the apply error route. + */ + streamed_entry = hash_search(streamingXactSize, + (void *) &xid, + HASH_FIND, &found); + /* + * It's possible that we get stream abort + * earlier than any call of write_stream_change that + * creates one hash entry for this xid. In this case, + * to find a entry with this xid fails. So just check + * if we've found it. Only when we confirm some writes + * by write_stream_change, report the stream_abort. + */ + if (found) + { + pgstat_report_subworker_xact_end(MyLogicalRepWorker->subid, + InvalidOid, + LOGICAL_REP_MSG_STREAM_ABORT, + streamed_entry->xact_size); + (void) hash_search(streamingXactSize, + (void *) &xid, + HASH_REMOVE, NULL); + } } else { @@ -1417,6 +1527,7 @@ apply_handle_stream_commit(StringInfo s) { TransactionId xid; LogicalRepCommitData commit_data; + XactSizeEntry *streamed_entry; if (in_streamed_transaction) ereport(ERROR, @@ -1438,6 +1549,19 @@ apply_handle_stream_commit(StringInfo s) /* Process any tables that are being synchronized in parallel. */ process_syncing_tables(commit_data.end_lsn); + /* Report and clean up the xid */ + streamed_entry = hash_search(streamingXactSize, + (void *) &xid, + HASH_FIND, NULL); + Assert(streamed_entry); + pgstat_report_subworker_xact_end(MyLogicalRepWorker->subid, + InvalidOid, + LOGICAL_REP_MSG_STREAM_COMMIT, + streamed_entry->xact_size); + (void) hash_search(streamingXactSize, + (void *) &xid, + HASH_REMOVE, NULL); + pgstat_report_activity(STATE_IDLE, NULL); reset_apply_error_context_info(); @@ -1576,6 +1700,8 @@ apply_handle_insert(StringInfo s) slot_fill_defaults(rel, estate, remoteslot); MemoryContextSwitchTo(oldctx); + update_apply_change_size(LOGICAL_REP_MSG_INSERT, rel); + /* For a partitioned table, insert the tuple into a partition. */ if (rel->localrel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE) apply_handle_tuple_routing(edata, @@ -1733,6 +1859,8 @@ apply_handle_update(StringInfo s) has_oldtup ? &oldtup : &newtup); MemoryContextSwitchTo(oldctx); + update_apply_change_size(LOGICAL_REP_MSG_UPDATE, rel); + /* For a partitioned table, apply update to correct partition. */ if (rel->localrel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE) apply_handle_tuple_routing(edata, @@ -1867,6 +1995,8 @@ apply_handle_delete(StringInfo s) slot_store_data(remoteslot, rel, &oldtup); MemoryContextSwitchTo(oldctx); + update_apply_change_size(LOGICAL_REP_MSG_DELETE, rel); + /* For a partitioned table, apply delete to correct partition. */ if (rel->localrel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE) apply_handle_tuple_routing(edata, @@ -2418,6 +2548,117 @@ apply_dispatch(StringInfo s) } /* + * Subscriber side implementation equivalent to ReorderBufferChangeSize + * of the publisher. + * + * According to the logical replication message type, record major + * resource consumptions of this subscription for each message. + * At present, do not collect data from generic functions to keep + * code simplicity, since the implementation complexity versus benefit + * tradeoff should not be good. Also, add multiple values + * at once in order to reduce the number of this function call. + * + * 'data' controls detail handling of data size calculation. + */ +void +update_apply_change_size(LogicalRepMsgType action, void *data) +{ + int64 size = 0; + LogicalRepRelMapEntry *relmapentry; + LogicalRepRelation *reprelation; + int *stream_write_len; + + /* + * In streaming mode, stream_write_change is called + * instead of immediate apply. List up the messages types + * that can be caught by handle_streamed_transaction and + * treat the write length as the size of transaction so + * that we can export it as part of pg_stat_subscription_worker. + */ + if (in_streamed_transaction && + (action == LOGICAL_REP_MSG_INSERT || + action == LOGICAL_REP_MSG_UPDATE || + action == LOGICAL_REP_MSG_DELETE || + action == LOGICAL_REP_MSG_TRUNCATE || + action == LOGICAL_REP_MSG_RELATION || + action == LOGICAL_REP_MSG_TYPE)) + { + stream_write_len = (int *) data; + size += *stream_write_len; + add_apply_error_context_xact_size(size); + return; + } + + switch (action) + { + /* No special memory consumption */ + case LOGICAL_REP_MSG_BEGIN: + case LOGICAL_REP_MSG_COMMIT: + case LOGICAL_REP_MSG_TRUNCATE: + case LOGICAL_REP_MSG_TYPE: + case LOGICAL_REP_MSG_ORIGIN: + case LOGICAL_REP_MSG_MESSAGE: + case LOGICAL_REP_MSG_STREAM_STOP: + case LOGICAL_REP_MSG_STREAM_ABORT: + case LOGICAL_REP_MSG_BEGIN_PREPARE: + break; + + case LOGICAL_REP_MSG_INSERT: + case LOGICAL_REP_MSG_UPDATE: + case LOGICAL_REP_MSG_DELETE: + Assert(data != NULL); + + /* + * Compute size based on ApplyExecutionData. + * The size of LogicalRepRelMapEntry can be skipped because + * it is obtained from hash_search in logicalrep_rel_open. + */ + size += sizeof(ApplyExecutionData) + sizeof(EState) + + sizeof(ResultRelInfo) + sizeof(ResultRelInfo); + + /* + * Add some extra size if the target relation is partitioned. + * PartitionTupleRouting isn't exported. Therefore, call the + * function that returns its size instead. + */ + relmapentry = (LogicalRepRelMapEntry *) data; + if (relmapentry->localrel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE) + size += sizeof(ModifyTableState) + PartitionTupleRoutingSize(); + break; + + case LOGICAL_REP_MSG_RELATION: + Assert(data != NULL); + + reprelation = (LogicalRepRelation *) data; + /* See logicalrep_read_attrs for the last two */ + size += sizeof(LogicalRepRelation) + + reprelation->natts * sizeof(char *) + + reprelation->natts * sizeof(Oid); + break; + + case LOGICAL_REP_MSG_STREAM_START: + size += sizeof(FileSet); + break; + + case LOGICAL_REP_MSG_PREPARE: + case LOGICAL_REP_MSG_COMMIT_PREPARED: + case LOGICAL_REP_MSG_ROLLBACK_PREPARED: + case LOGICAL_REP_MSG_STREAM_COMMIT: + case LOGICAL_REP_MSG_STREAM_PREPARE: + size += sizeof(FlushPosition); + break; + + default: + ereport(ERROR, + (errcode(ERRCODE_PROTOCOL_VIOLATION), + errmsg("invalid logical replication message type \"%c\"", action))); + } + + /* update the total size of consumption */ + add_apply_error_context_xact_size(size); +} + +/* * Figure out which write/flush positions to report to the walsender process. * * We can't simply report back the last LSN the walsender sent us because the @@ -3271,6 +3512,9 @@ static void stream_write_change(char action, StringInfo s) { int len; + int total_len; + bool found; + XactSizeEntry *streamed_entry; Assert(in_streamed_transaction); Assert(TransactionIdIsValid(stream_xid)); @@ -3289,6 +3533,16 @@ stream_write_change(char action, StringInfo s) len = (s->len - s->cursor); BufFileWrite(stream_fd, &s->data[s->cursor], len); + + /* update xact size by xid */ + total_len = (s->len - s->cursor) * 2 + sizeof(char) + sizeof(action); + streamed_entry = (XactSizeEntry *) hash_search(streamingXactSize, + (void *) &stream_xid, + HASH_ENTER, &found); + if (!found) + streamed_entry->xact_size = total_len; /* init */ + else + streamed_entry->xact_size = streamed_entry->xact_size + total_len; /* update */ } /* @@ -3426,6 +3680,23 @@ ApplyWorkerMain(Datum main_arg) elog(DEBUG1, "connecting to publisher using connection string \"%s\"", MySubscription->conninfo); + /* + * Initialize the apply worker's hash to manage bytes for + * streaming txns. + */ + if (!am_tablesync_worker() && MySubscription->stream) + { + HASHCTL hash_ctl; + + hash_ctl.keysize = sizeof(TransactionId); + hash_ctl.entrysize = sizeof(XactSizeEntry); + hash_ctl.hcxt = LogicalStreamingContext; + streamingXactSize = hash_create("xact size per streaming xid", + PARALLEL_STREAMING_XACTS, + &hash_ctl, + HASH_ELEM | HASH_BLOBS | HASH_CONTEXT); + } + if (am_tablesync_worker()) { char *syncslotname; @@ -3601,6 +3872,27 @@ ApplyWorkerMain(Datum main_arg) proc_exit(0); } +/* Exported so that stats collector can utilize this value */ +int64 +get_apply_error_context_xact_size(void) +{ + return apply_error_callback_arg.bytes; +} + +/* Add size to apply error bytes */ +void +add_apply_error_context_xact_size(int64 size) +{ + apply_error_callback_arg.bytes += size; +} + +/* Reset information of apply error callback */ +void +reset_apply_error_context_xact_size(void) +{ + apply_error_callback_arg.bytes = 0; +} + /* * Is current process a logical replication worker? */ diff --git a/src/backend/utils/adt/pgstatfuncs.c b/src/backend/utils/adt/pgstatfuncs.c index 2511df1..99b3e7f 100644 --- a/src/backend/utils/adt/pgstatfuncs.c +++ b/src/backend/utils/adt/pgstatfuncs.c @@ -2405,7 +2405,7 @@ pg_stat_get_replication_slot(PG_FUNCTION_ARGS) Datum pg_stat_get_subscription_worker(PG_FUNCTION_ARGS) { -#define PG_STAT_GET_SUBSCRIPTION_WORKER_COLS 9 +#define PG_STAT_GET_SUBSCRIPTION_WORKER_COLS 15 Oid subid = PG_GETARG_OID(0); Oid subrelid; TupleDesc tupdesc; @@ -2425,19 +2425,31 @@ pg_stat_get_subscription_worker(PG_FUNCTION_ARGS) OIDOID, -1, 0); TupleDescInitEntry(tupdesc, (AttrNumber) 2, "subrelid", OIDOID, -1, 0); - TupleDescInitEntry(tupdesc, (AttrNumber) 3, "relid", + TupleDescInitEntry(tupdesc, (AttrNumber) 3, "xact_commit", + INT8OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) 4, "xact_commit_bytes", + INT8OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) 5, "xact_error", + INT8OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) 6, "xact_error_bytes", + INT8OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) 7, "xact_abort", + INT8OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) 8, "xact_abort_bytes", + INT8OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) 9, "last_error_relid", OIDOID, -1, 0); - TupleDescInitEntry(tupdesc, (AttrNumber) 4, "command", + TupleDescInitEntry(tupdesc, (AttrNumber) 10, "last_error_command", TEXTOID, -1, 0); - TupleDescInitEntry(tupdesc, (AttrNumber) 5, "xid", + TupleDescInitEntry(tupdesc, (AttrNumber) 11, "last_error_xid", XIDOID, -1, 0); - TupleDescInitEntry(tupdesc, (AttrNumber) 6, "error_count", + TupleDescInitEntry(tupdesc, (AttrNumber) 12, "last_error_count", INT8OID, -1, 0); - TupleDescInitEntry(tupdesc, (AttrNumber) 7, "error_message", + TupleDescInitEntry(tupdesc, (AttrNumber) 13, "last_error_message", TEXTOID, -1, 0); - TupleDescInitEntry(tupdesc, (AttrNumber) 8, "last_error_time", + TupleDescInitEntry(tupdesc, (AttrNumber) 14, "last_error_time", TIMESTAMPTZOID, -1, 0); - TupleDescInitEntry(tupdesc, (AttrNumber) 9, "stats_reset", + TupleDescInitEntry(tupdesc, (AttrNumber) 15, "stats_reset", TIMESTAMPTZOID, -1, 0); BlessTupleDesc(tupdesc); @@ -2462,28 +2474,36 @@ pg_stat_get_subscription_worker(PG_FUNCTION_ARGS) else nulls[i++] = true; - /* relid */ + /* transaction stats */ + values[i++] = Int64GetDatum(wentry->xact_commit); + values[i++] = Int64GetDatum(wentry->xact_commit_bytes); + values[i++] = Int64GetDatum(wentry->xact_error); + values[i++] = Int64GetDatum(wentry->xact_error_bytes); + values[i++] = Int64GetDatum(wentry->xact_abort); + values[i++] = Int64GetDatum(wentry->xact_abort_bytes); + + /* last_error_relid */ if (OidIsValid(wentry->relid)) values[i++] = ObjectIdGetDatum(wentry->relid); else nulls[i++] = true; - /* command */ + /* last_error_command */ if (wentry->command != 0) values[i++] = CStringGetTextDatum(logicalrep_message_type(wentry->command)); else nulls[i++] = true; - /* xid */ + /* last_error_xid */ if (TransactionIdIsValid(wentry->xid)) values[i++] = TransactionIdGetDatum(wentry->xid); else nulls[i++] = true; - /* error_count */ + /* last_error_count */ values[i++] = Int64GetDatum(wentry->error_count); - /* error_message */ + /* last_error_message */ values[i++] = CStringGetTextDatum(wentry->error_message); /* last_error_time */ diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index e6c7abb..db7d270 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -5389,9 +5389,9 @@ proname => 'pg_stat_get_subscription_worker', prorows => '1', proisstrict => 'f', proretset => 't', provolatile => 's', proparallel => 'r', prorettype => 'record', proargtypes => 'oid oid', - proallargtypes => '{oid,oid,oid,oid,oid,text,xid,int8,text,timestamptz,timestamptz}', - proargmodes => '{i,i,o,o,o,o,o,o,o,o,o}', - proargnames => '{subid,subrelid,subid,subrelid,relid,command,xid,error_count,error_message,last_error_time,stats_reset}', + proallargtypes => '{oid,oid,oid,oid,int8,int8,int8,int8,int8,int8,oid,text,xid,int8,text,timestamptz,timestamptz}', + proargmodes => '{i,i,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o}', + proargnames => '{subid,subrelid,subid,subrelid,xact_commit,xact_commit_bytes,xact_error,xact_error_bytes,xact_abort,xact_abort_bytes,last_error_relid,last_error_command,last_error_xid,last_error_count,last_error_message,last_error_time,stats_reset}', prosrc => 'pg_stat_get_subscription_worker' }, { oid => '6118', descr => 'statistics: information about subscription', proname => 'pg_stat_get_subscription', prorows => '10', proisstrict => 'f', diff --git a/src/include/executor/execPartition.h b/src/include/executor/execPartition.h index 694e38b..773e46c 100644 --- a/src/include/executor/execPartition.h +++ b/src/include/executor/execPartition.h @@ -110,6 +110,7 @@ typedef struct PartitionPruneState PartitionPruningData *partprunedata[FLEXIBLE_ARRAY_MEMBER]; } PartitionPruneState; +extern size_t PartitionTupleRoutingSize(void); extern PartitionTupleRouting *ExecSetupPartitionTupleRouting(EState *estate, Relation rel); extern ResultRelInfo *ExecFindPartition(ModifyTableState *mtstate, diff --git a/src/include/pgstat.h b/src/include/pgstat.h index 7a26d6d..aefaf45 100644 --- a/src/include/pgstat.h +++ b/src/include/pgstat.h @@ -88,6 +88,8 @@ typedef enum StatMsgType PGSTAT_MTYPE_SUBSCRIPTIONPURGE, PGSTAT_MTYPE_SUBWORKERERROR, PGSTAT_MTYPE_SUBWORKERPURGE, + PGSTAT_MTYPE_SUBWORKERXACTEND, + PGSTAT_MTYPE_SUBWORKERTWOPHASEXACT, } StatMsgType; /* ---------- @@ -591,6 +593,54 @@ typedef struct PgStat_MsgSubWorkerPurge } PgStat_MsgSubWorkerPurge; /* ---------- + * PgStat_MsgSubscriptionXactEnd Sent by the apply worker or the table sync worker + * to report successful transaction ends. + * ---------- + */ +typedef struct PgStat_MsgSubWorkerXactEnd +{ + PgStat_MsgHdr m_hdr; + + /* determine the worker entry */ + Oid m_subid; + Oid m_subrelid; + + /* + * distinguish between transaction commits and streaming transaction aborts + * that are handled without error. + */ + LogicalRepMsgType m_command; + + /* memory consumption used by transaction */ + PgStat_Counter m_xact_bytes; + +} PgStat_MsgSubWorkerXactEnd; + +/* ---------- + * PgStat_MsgSubWorkerTwophaseXact Sent by the apply worker to make size of prepared + * txn persistent over the server restart and make it + * visible after commit prepare or rollback prepared. + * This is separated from PgStat_MsgSubWorkerXactEnd + * so that we can reduce message size of gid for other + * operations (e.g. normal COMMIT) that should happen more + * frequently than prepare operation usually. + * ---------- + */ +typedef struct PgStat_MsgSubWorkerTwophaseXact +{ + PgStat_MsgHdr m_hdr; + + /* determine the subscription */ + Oid m_subid; + + LogicalRepMsgType m_command; + char m_gid[GIDSIZE]; + int gid_len; + PgStat_Counter m_xact_bytes; + +} PgStat_MsgSubWorkerTwophaseXact; + +/* ---------- * PgStat_MsgSubWorkerError Sent by the apply worker or the table sync worker to * report the error occurred during logical replication. * ---------- @@ -609,6 +659,12 @@ typedef struct PgStat_MsgSubWorkerError Oid m_subrelid; /* + * Transaction stats of subscription needs to be updated when an + * error occurs. + */ + PgStat_Counter m_xact_error_bytes; + + /* * Oids of the database and the table that the reporter was actually * processing. m_relid can be InvalidOid if an error occurred during * worker applying a non-data-modification message such as RELATION. @@ -803,6 +859,8 @@ typedef union PgStat_Msg PgStat_MsgSubscriptionPurge msg_subscriptionpurge; PgStat_MsgSubWorkerError msg_subworkererror; PgStat_MsgSubWorkerPurge msg_subworkerpurge; + PgStat_MsgSubWorkerXactEnd msg_subworkerxactend; + PgStat_MsgSubWorkerTwophaseXact msg_subworkertwophasexact; } PgStat_Msg; @@ -1035,6 +1093,16 @@ typedef struct PgStat_StatSubWorkerEntry PgStat_StatSubWorkerKey key; /* hash key (must be first) */ /* + * Transaction statistics of subscription worker + */ + PgStat_Counter xact_commit; + PgStat_Counter xact_commit_bytes; + PgStat_Counter xact_error; + PgStat_Counter xact_error_bytes; + PgStat_Counter xact_abort; + PgStat_Counter xact_abort_bytes; + + /* * Subscription worker error statistics representing an error that * occurred during application of logical replication or the initial table * synchronization. @@ -1049,6 +1117,22 @@ typedef struct PgStat_StatSubWorkerEntry TimestampTz stat_reset_timestamp; } PgStat_StatSubWorkerEntry; +/* prepared transaction */ +typedef struct PgStat_StatSubWorkerPreparedXact +{ + Oid subid; + char gid[GIDSIZE]; +} PgStat_StatSubWorkerPreparedXact; + +typedef struct PgStat_StatSubWorkerPreparedXactSize +{ + PgStat_StatSubWorkerPreparedXact key; /* hash key */ + + Oid subid; + char gid[GIDSIZE]; + PgStat_Counter xact_size; +} PgStat_StatSubWorkerPreparedXactSize; + /* * Working state needed to accumulate per-function-call timing statistics. */ @@ -1158,6 +1242,13 @@ extern void pgstat_report_checksum_failure(void); extern void pgstat_report_replslot(const PgStat_StatReplSlotEntry *repSlotStat); extern void pgstat_report_replslot_create(const char *slotname); extern void pgstat_report_replslot_drop(const char *slotname); +extern void pgstat_report_subworker_xact_end(Oid subid, Oid subrel, + LogicalRepMsgType command, PgStat_Counter xact_size); +extern void pgstat_report_subworker_twophase_xact(Oid subid, LogicalRepMsgType command, + PgStat_Counter xact_size, + LogicalRepPreparedTxnData *prepared_data, + LogicalRepCommitPreparedTxnData *commit_data, + LogicalRepRollbackPreparedTxnData *rollback_data); extern void pgstat_report_subworker_error(Oid subid, Oid subrelid, Oid relid, LogicalRepMsgType command, TransactionId xid, const char *errmsg); diff --git a/src/include/replication/logicalworker.h b/src/include/replication/logicalworker.h index 2ad61a0..9a8447b 100644 --- a/src/include/replication/logicalworker.h +++ b/src/include/replication/logicalworker.h @@ -16,4 +16,9 @@ extern void ApplyWorkerMain(Datum main_arg); extern bool IsLogicalWorker(void); +/* for transaction stats */ +extern int64 get_apply_error_context_xact_size(void); +extern void add_apply_error_context_xact_size(int64 size); +extern void reset_apply_error_context_xact_size(void); + #endif /* LOGICALWORKER_H */ diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out index f6b1bd6..a5eb7c3 100644 --- a/src/test/regress/expected/rules.out +++ b/src/test/regress/expected/rules.out @@ -2097,11 +2097,17 @@ pg_stat_subscription| SELECT su.oid AS subid, pg_stat_subscription_workers| SELECT w.subid, s.subname, w.subrelid, - w.relid, - w.command, - w.xid, - w.error_count, - w.error_message, + w.xact_commit, + w.xact_commit_bytes, + w.xact_error, + w.xact_error_bytes, + w.xact_abort, + w.xact_abort_bytes, + w.last_error_relid, + w.last_error_command, + w.last_error_xid, + w.last_error_count, + w.last_error_message, w.last_error_time, w.stats_reset FROM ( SELECT pg_subscription.oid AS subid, @@ -2112,7 +2118,7 @@ pg_stat_subscription_workers| SELECT w.subid, pg_subscription_rel.srrelid AS relid FROM pg_subscription_rel WHERE (pg_subscription_rel.srsubstate <> 'r'::"char")) sr, - (LATERAL pg_stat_get_subscription_worker(sr.subid, sr.relid) w(subid, subrelid, relid, command, xid, error_count, error_message, last_error_time, stats_reset) + (LATERAL pg_stat_get_subscription_worker(sr.subid, sr.relid) w(subid, subrelid, xact_commit, xact_commit_bytes, xact_error, xact_error_bytes, xact_abort, xact_abort_bytes, last_error_relid, last_error_command, last_error_xid, last_error_count, last_error_message, last_error_time, stats_reset) JOIN pg_subscription s ON ((w.subid = s.oid))); pg_stat_sys_indexes| SELECT pg_stat_all_indexes.relid, pg_stat_all_indexes.indexrelid, diff --git a/src/test/subscription/t/026_error_report.pl b/src/test/subscription/t/026_error_report.pl index 3d23bb5..9ff619e 100644 --- a/src/test/subscription/t/026_error_report.pl +++ b/src/test/subscription/t/026_error_report.pl @@ -15,8 +15,8 @@ sub test_subscription_error my $check_sql = qq[ SELECT count(1) > 0 FROM pg_stat_subscription_workers -WHERE relid = '$relname'::regclass]; - $check_sql .= " AND xid = '$xid'::xid;" if $xid ne ''; +WHERE last_error_relid = '$relname'::regclass]; + $check_sql .= " AND last_error_xid = '$xid'::xid;" if $xid ne ''; # Wait for the error statistics to be updated. $node->poll_query_until( @@ -26,9 +26,9 @@ WHERE relid = '$relname'::regclass]; my $result = $node->safe_psql( 'postgres', qq[ -SELECT subname, command, relid::regclass, error_count > 0 +SELECT subname, last_error_command, last_error_relid::regclass, last_error_count > 0 FROM pg_stat_subscription_workers -WHERE relid = '$relname'::regclass; +WHERE last_error_relid = '$relname'::regclass; ]); is($result, $expected_error, $msg); } diff --git a/src/test/subscription/t/027_worker_xact_stats.pl b/src/test/subscription/t/027_worker_xact_stats.pl new file mode 100644 index 0000000..9b3f140 --- /dev/null +++ b/src/test/subscription/t/027_worker_xact_stats.pl @@ -0,0 +1,174 @@ + +# Copyright (c) 2021, PostgreSQL Global Development Group + +# Tests for subscription worker statistics during apply. +use strict; +use warnings; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More tests => 3; + +# Create publisher node +my $node_publisher = PostgreSQL::Test::Cluster->new('publisher'); +$node_publisher->init(allows_streaming => 'logical'); +$node_publisher->append_conf('postgresql.conf', qq[ +logical_decoding_work_mem = 64kB +max_prepared_transactions = 10 +max_wal_senders = 10 +wal_sender_timeout = 0 +]); +$node_publisher->start; + +# Create subscriber node +my $node_subscriber = PostgreSQL::Test::Cluster->new('subscriber'); +$node_subscriber->init(allows_streaming => 'logical'); +$node_subscriber->append_conf('postgresql.conf', qq[ +max_prepared_transactions = 10 +]); +$node_subscriber->start; + +# Setup structures on the publisher and the subscriber +$node_publisher->safe_psql('postgres', + "CREATE TABLE test_tab (a int)"); +$node_subscriber->safe_psql('postgres', + "CREATE TABLE test_tab (a int primary key);"); + +# Setup logical replication +my $publisher_connstr = $node_publisher->connstr . ' dbname=postgres'; +$node_publisher->safe_psql('postgres', + "CREATE PUBLICATION tap_pub FOR TABLE test_tab"); + +my $appname = 'tap_sub'; +$node_subscriber->safe_psql('postgres', qq[ +CREATE SUBSCRIPTION tap_sub +CONNECTION '$publisher_connstr application_name=$appname' +PUBLICATION tap_pub WITH (streaming = on, two_phase = on, copy_data = off); +]); + +$node_publisher->wait_for_catchup($appname); + +# There's no entry at the beginning +my $result = $node_subscriber->safe_psql('postgres', +"SELECT count(*) FROM pg_stat_subscription_workers;"); +is($result, '0', 'no entry for transaction stats yet'); + +# COMMIT +$node_publisher->safe_psql('postgres', + "BEGIN; INSERT INTO test_tab VALUES (1); COMMIT;"); + +$node_subscriber->poll_query_until('postgres', +"SELECT count(1) = 1 FROM pg_stat_subscription_workers where xact_commit = 1;") + or die "didn't get updates of xact stats by commit"; + +# Now, stats collector make the bytes updated also. +$result = $node_subscriber->safe_psql('postgres', +"SELECT xact_commit_bytes > 0 FROM pg_stat_subscription_workers;"); +is($result, 't', 'got consumed bytes'); + +# STREAM COMMIT +$node_publisher->safe_psql( + 'postgres', + "BEGIN; INSERT INTO test_tab VALUES(generate_series(1001, 2000)); COMMIT;"); +$result = $node_publisher->safe_psql('postgres', +"SELECT stream_count > 0 FROM pg_stat_replication_slots;"); +is($result, 't', "confirm the streaming has happened."); +$node_subscriber->poll_query_until('postgres', +"SELECT count(1) = 1 FROM pg_stat_subscription_workers where xact_commit = 2;") + or die "didn't get updates of xact stats by stream commit"; + +# STREAM PREPARE & COMMIT PREPARED +# This should restore the xact size before the shutdown. +$node_publisher->safe_psql( + 'postgres', q[ +BEGIN; +INSERT INTO test_tab VALUES(generate_series(2001, 3000)); +PREPARE TRANSACTION 'gid1'; +]); + +# This streamed prepare is not displayed until the commit prepared +# or rollback prepared. Hence, there's no way to confirm that +# stats collector has received the bytes of prepared transaction. +# So, instead of checking the view, issue one more committed transaction +# after the prepare and make sure that this commit's update is done, +# which should mean the previous streamed prepare is already processed +# by the stats collector as well. +$node_publisher->safe_psql('postgres', +"BEGIN; INSERT INTO test_tab VALUES (2); COMMIT;"); +$node_subscriber->poll_query_until('postgres', +"SELECT count(1) = 1 FROM pg_stat_subscription_workers where xact_commit = 3;") + or die "didn't process the updates of committed transaction"; + +$node_subscriber->restart; + +# Get xact_commit_bytes before commit prepared. +my $tmp = $node_subscriber->safe_psql('postgres', +"SELECT xact_commit_bytes FROM pg_stat_subscription_workers where xact_commit = 3;"); + +# Commit prepared increments the xact_commit. +$node_publisher->safe_psql('postgres', "COMMIT PREPARED 'gid1'"); +$node_subscriber->poll_query_until('postgres', +"SELECT count(1) = 1 FROM pg_stat_subscription_workers where xact_commit = 4;") + or die "didn't get updates of xact stats by stream prepare and commit prepared"; + +$node_subscriber->poll_query_until('postgres', +"SELECT xact_commit_bytes > $tmp FROM pg_stat_subscription_workers where xact_commit = 4;"); + +# STREAM ABORT +# Store previous stream_counts to recognize +# another streaming from the increase of this value below. +$tmp = $node_publisher->safe_psql('postgres', +"SELECT stream_count FROM pg_stat_replication_slots"); + +# Cause a new streaming and check it by another session +my $in = ''; +my $out = ''; +my $timer = IPC::Run::timeout(180); +my $h = $node_publisher->background_psql('postgres', \$in, \$out, $timer, + on_error_stop => 1); + +$in .= q{ +BEGIN; +INSERT INTO test_tab VALUES(generate_series(3001, 4000)); +}; +$h->pump_nb; + +# Wait until this transaction is streamed certainly +# and after that rollback to send stream abort. +$node_publisher->poll_query_until('postgres', +"SELECT stream_count > $tmp FROM pg_stat_replication_slots;") + or die "didn't stream data of a new transaction"; + +$in .= q{ +ROLLBACK; +\q +}; +$h->finish; + +$node_subscriber->poll_query_until('postgres', +"SELECT count(1) = 1 FROM pg_stat_subscription_workers where xact_abort = 1;") + or die "didn't get updates of xact stats by stream abort"; + +# ROLLBACK PREPARED +$node_publisher->safe_psql('postgres', q[ +BEGIN; +INSERT INTO test_tab VALUES (3); +PREPARE TRANSACTION 'gid2'; +]); +$node_publisher->safe_psql('postgres', "ROLLBACK PREPARED 'gid2'"); +$node_subscriber->poll_query_until('postgres', +"SELECT count(1) = 1 FROM pg_stat_subscription_workers where xact_abort = 2;") + or die "didn't get updates of xact stats by rollback prepared"; + +# error stats (by duplication error) +$node_publisher->safe_psql('postgres', q[ +BEGIN; +INSERT INTO test_tab VALUES (1); +COMMIT; +]); + +$node_subscriber->poll_query_until('postgres', +"SELECT count(1) = 1 FROM pg_stat_subscription_workers where xact_error > 0;") + or die "didn't get updates of xact stats by error"; + +$node_subscriber->stop('fast'); +$node_publisher->stop('fast'); ^ permalink raw reply [nested|flat] 113+ messages in thread
* RE: Failed transaction statistics to measure the logical replication progress @ 2021-11-02 12:07 [email protected] <[email protected]> parent: [email protected] <[email protected]> 1 sibling, 0 replies; 113+ messages in thread From: [email protected] @ 2021-11-02 12:07 UTC (permalink / raw) To: 'Masahiko Sawada' <[email protected]>; Amit Kapila <[email protected]>; +Cc: [email protected] <[email protected]>; [email protected] <[email protected]> On Monday, November 1, 2021 10:18 PM I wrote: > On Thursday, October 28, 2021 11:19 PM I wrote: > > I've created a new patch that extends pg_stat_subscription_workers to > > include other transaction statistics. > > > > Note that this patch depends on v18 patch-set in [1]... > Rebased based on the v19 in [1]. I'd like to add one more explanation about the latest patches for review. On Thursday, October 28, 2021 11:19 PM I wrote: > (2) > Updates of stats on the view at either commit prepared or rollback prepared > time. > This means we don't lost prepared transaction size even after server restart > and user can see the results of two phase operation at those timings. There is another reason I had to treat 'prepare' message like above. Any reviewer might think that sending prepared bytes to stats collector and making the bytes survive over the restart is too much. But, we don't know if the prepared transaction is processed by commit prepared or rollback prepared at 'prepare' time. An execution of commit prepared needs to update the column of xact_commit and xact_commit_bytes while rollback prepared does the column of xact_abort and xact_abort_bytes where this patch is introducing. Therefore, even when we can calculate the bytes of prepared txn at prepare time, it's *not* possible to add the transaction bytes to either stats column of bytes sizes and clean up the bytes. Then, there was a premise that we should not lose the prepared txn bytes by the shutdown so my patch has become the implementation described above. Best Regards, Takamichi Osumi ^ permalink raw reply [nested|flat] 113+ messages in thread
* Re: Failed transaction statistics to measure the logical replication progress @ 2021-11-04 00:53 Greg Nancarrow <[email protected]> parent: [email protected] <[email protected]> 1 sibling, 1 reply; 113+ messages in thread From: Greg Nancarrow @ 2021-11-04 00:53 UTC (permalink / raw) To: [email protected] <[email protected]>; +Cc: Masahiko Sawada <[email protected]>; Amit Kapila <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]> On Tue, Nov 2, 2021 at 12:18 AM [email protected] <[email protected]> wrote: > > On Thursday, October 28, 2021 11:19 PM I wrote: > > I've created a new patch that extends pg_stat_subscription_workers to include > > other transaction statistics. > > > > Note that this patch depends on v18 patch-set in [1]... > Rebased based on the v19 in [1]. > Also, fixed documentation a little and made tests tidy. > FYI, the newly included TAP test(027_worker_xact_stats.pl) is stable > because I checked that 100 times of its execution in a tight loop all passed. > I have done some basic testing of the patch and have some initial review comments: (1) I think this patch needs to be in "git format-patch" format, with a proper commit message that describes the purpose of the patch and the functionality it adds, and any high-level design points (something like the overview given in the initial post, and accounting for the subsequent discussion points and updated functionality). (2) doc/src/sgml/monitoring.sgml There are some grammatical issues in the current description. I suggest changing it to something like: BEFORE: + <entry>At least one row per subscription, showing about transaction statistics and error summary that AFTER: + <entry>At least one row per subscription, showing transaction statistics and information about errors that (2) doc/src/sgml/monitoring.sgml The current description seems a little confusing. Per subscription, it shows the transaction statistics and any last error info from tablesync/apply workers? If this is the case, I'd suggest the following change: BEFORE: + one row per subscription for transaction statistics and summary of the last + error reported by workers applying logical replication changes and workers + handling the initial data copy of the subscribed tables. AFTER: + one row per subscription, showing corresponding transaction statistics and + information about the last error reported by workers applying logical replication + changes or by workers handling the initial data copy of the subscribed tables. (3) xact_commit I think that the "xact_commit" column should be named "xact_commit_count" or "xact_commits". Similarly, I think "xact_error" should be named "xact_error_count" or "xact_errors", and "xact_aborts" should be named "xact_abort_count" or "xact_aborts". (4) xact_commit_bytes + Amount of transactions data successfully applied in this subscription. + Consumed memory for xact_commit is displayed. I find this description a bit confusing. "Consumed memory for xact_commit" seems different to "transactions data". Could the description be something like: Amount of data (in bytes) successfully applied in this subscription, across "xact_commit_count" transactions. (5) I'd suggest some minor rewording for the following: BEFORE: + Number of transactions failed to be applied and caught by table + sync worker or main apply worker in this subscription. AFTER: + Number of transactions that failed to be applied by the table + sync worker or main apply worker in this subscription. (6) xact_error_bytes Again, it's a little confusing referring to "consumed memory" here. How about rewording this, something like: BEFORE: + Amount of transactions data unsuccessfully applied in this subscription. + Consumed memory that past failed transaction used is displayed. AFTER: + Amount of data (in bytes) unsuccessfully applied in this subscription by the last failed transaction. (7) The additional information provided for "xact_abort_bytes" needs some rewording, something like: BEFORE: + Increase <literal>logical_decoding_work_mem</literal> on the publisher + so that it exceeds the size of whole streamed transaction + to suppress unnecessary consumed network bandwidth in addition to change + in memory of the subscriber, if unexpected amount of streamed transactions + are aborted. AFTER: + In order to suppress unnecessary consumed network bandwidth, increase + <literal>logical_decoding_work_mem</literal> on the publisher so that it + exceeds the size of the whole streamed transaction, and additionally increase + the available subscriber memory, if an unexpected amount of streamed transactions + are aborted. (8) Suggested update: BEFORE: + * Tell the collector that worker transaction has finished without problem. AFTER: + * Tell the collector that the worker transaction has successfully completed. (9) src/backend/postmaster/pgstat.c I think that the GID copying is unnecessarily copying the whole GID buffer or using an additional strlen(). It should be changed to use strlcpy() to match other code: BEFORE: + /* get the gid for this two phase operation */ + if (command == LOGICAL_REP_MSG_PREPARE || + command == LOGICAL_REP_MSG_STREAM_PREPARE) + memcpy(msg.m_gid, prepare_data->gid, GIDSIZE); + else if (command == LOGICAL_REP_MSG_COMMIT_PREPARED) + memcpy(msg.m_gid, commit_data->gid, GIDSIZE); + else /* rollback prepared */ + memcpy(msg.m_gid, rollback_data->gid, GIDSIZE); AFTER: + /* get the gid for this two phase operation */ + if (command == LOGICAL_REP_MSG_PREPARE || + command == LOGICAL_REP_MSG_STREAM_PREPARE) + strlcpy(msg.m_gid, prepare_data->gid, sizeof(msg.m_gid)); + else if (command == LOGICAL_REP_MSG_COMMIT_PREPARED) + strlcpy(msg.m_gid, commit_data->gid, sizeof(msg.m_gid)); + else /* rollback prepared */ + strlcpy(msg.m_gid, rollback_data->gid, sizeof(msg.m_gid)); BEFORE: + strlcpy(prepared_txn->gid, msg->m_gid, strlen(msg->m_gid) + 1); AFTER: + strlcpy(prepared_txn->gid, msg->m_gid, sizeof(prepared_txn->gid)); BEFORE: + memcpy(key.gid, msg->m_gid, strlen(msg->m_gid)); AFTER: + strlcpy(key.gid, msg->m_gid, sizeof(key.gid)); BEFORE: + memcpy(key.gid, gid, strlen(gid)); AFTER: + strlcpy(key.gid, gid, sizeof(key.gid)); (10) src/backend/replication/logical/worker.c Some suggested rewording: BEFORE: + * size of streaming transaction resources because it have used the AFTER: + * size of streaming transaction resources because it has used the BEFORE: + * tradeoff should not be good. Also, add multiple values + * at once in order to reduce the number of this function call. AFTER: + * tradeoff would not be good. Also, add multiple values + * at once in order to reduce the number of calls to this function. (11) update_apply_change_size() Shouldn't this function be declared static? (12) stream_write_change() + streamed_entry->xact_size = streamed_entry->xact_size + total_len; /* update */ could be simply written as: + streamed_entry->xact_size += total_len; /* update */ Regards, Greg Nancarrow Fujitsu Australia ^ permalink raw reply [nested|flat] 113+ messages in thread
* RE: Failed transaction statistics to measure the logical replication progress @ 2021-11-05 08:11 [email protected] <[email protected]> parent: Greg Nancarrow <[email protected]> 0 siblings, 3 replies; 113+ messages in thread From: [email protected] @ 2021-11-05 08:11 UTC (permalink / raw) To: 'Greg Nancarrow' <[email protected]>; +Cc: Masahiko Sawada <[email protected]>; Amit Kapila <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]> On Thursday, November 4, 2021 9:54 AM Greg Nancarrow <[email protected]> wrote: > On Tue, Nov 2, 2021 at 12:18 AM [email protected] > <[email protected]> wrote: > > > > On Thursday, October 28, 2021 11:19 PM I wrote: > > > I've created a new patch that extends pg_stat_subscription_workers > > > to include other transaction statistics. > > > > > > Note that this patch depends on v18 patch-set in [1]... > > Rebased based on the v19 in [1]. > > Also, fixed documentation a little and made tests tidy. > > FYI, the newly included TAP test(027_worker_xact_stats.pl) is stable > > because I checked that 100 times of its execution in a tight loop all passed. > > > > I have done some basic testing of the patch and have some initial review > comments: Thanks for your review ! > (1) I think this patch needs to be in "git format-patch" format, with a proper > commit message that describes the purpose of the patch and the functionality > it adds, and any high-level design points (something like the overview given in > the initial post, and accounting for the subsequent discussion points and > updated functionality). Fixed. > (2) doc/src/sgml/monitoring.sgml > There are some grammatical issues in the current description. I suggest > changing it to something like: > BEFORE: > + <entry>At least one row per subscription, showing about > transaction statistics and error summary that > AFTER: > + <entry>At least one row per subscription, showing transaction > statistics and information about errors that Fixed. > (2) doc/src/sgml/monitoring.sgml > The current description seems a little confusing. > Per subscription, it shows the transaction statistics and any last error info from > tablesync/apply workers? If this is the case, I'd suggest the following change: > > BEFORE: > + one row per subscription for transaction statistics and summary of the > last > + error reported by workers applying logical replication changes and > workers > + handling the initial data copy of the subscribed tables. > AFTER: > + one row per subscription, showing corresponding transaction statistics > and > + information about the last error reported by workers applying > logical replication > + changes or by workers handling the initial data copy of the > subscribed tables. Fixed. > (3) xact_commit > I think that the "xact_commit" column should be named "xact_commit_count" > or "xact_commits". > Similarly, I think "xact_error" should be named "xact_error_count" or > "xact_errors", and "xact_aborts" should be named "xact_abort_count" or > "xact_aborts". I prefered *_count. Renamed. > (4) xact_commit_bytes > > + Amount of transactions data successfully applied in this subscription. > + Consumed memory for xact_commit is displayed. > > I find this description a bit confusing. "Consumed memory for xact_commit" > seems different to "transactions data". > Could the description be something like: Amount of data (in bytes) successfully > applied in this subscription, across "xact_commit_count" > transactions. Fixed. > (5) > I'd suggest some minor rewording for the following: > > BEFORE: > + Number of transactions failed to be applied and caught by table > + sync worker or main apply worker in this subscription. > AFTER: > + Number of transactions that failed to be applied by the table > + sync worker or main apply worker in this subscription. Fixed. > (6) xact_error_bytes > Again, it's a little confusing referring to "consumed memory" here. > How about rewording this, something like: > > BEFORE: > + Amount of transactions data unsuccessfully applied in this > subscription. > + Consumed memory that past failed transaction used is displayed. > AFTER: > + Amount of data (in bytes) unsuccessfully applied in this > subscription by the last failed transaction. xact_error_bytes (and other bytes columns as well) is cumulative so when a new error happens, the size of this new bytes would be added to the same. So here we shouldn't mention just the last error. I simply applied your previous comments of 'xact_commit_bytes' to 'xact_error_bytes' description. > (7) > The additional information provided for "xact_abort_bytes" needs some > rewording, something like: > > BEFORE: > + Increase <literal>logical_decoding_work_mem</literal> on the > publisher > + so that it exceeds the size of whole streamed transaction > + to suppress unnecessary consumed network bandwidth in addition to > change > + in memory of the subscriber, if unexpected amount of streamed > transactions > + are aborted. > AFTER: > + In order to suppress unnecessary consumed network bandwidth, > increase > + <literal>logical_decoding_work_mem</literal> on the publisher so > that it > + exceeds the size of the whole streamed transaction, and > additionally increase > + the available subscriber memory, if an unexpected amount of > streamed transactions > + are aborted. I'm not sure about the last part. > additionally increase the available subscriber memory, Which GUC parameter did you mean by this ? Could we point out and enalrge the memory size only for subscriber's apply processing intentionally ? I incorporated (7) except for this last part. Will revise according to your reply. I also added the explanation about xact_abort_bytes itself to align with other bytes columns. > (8) > Suggested update: > > BEFORE: > + * Tell the collector that worker transaction has finished without problem. > AFTER: > + * Tell the collector that the worker transaction has successfully completed. Fixed. > (9) src/backend/postmaster/pgstat.c > I think that the GID copying is unnecessarily copying the whole GID buffer or > using an additional strlen(). > It should be changed to use strlcpy() to match other code: > > BEFORE: > + /* get the gid for this two phase operation */ if (command == > + LOGICAL_REP_MSG_PREPARE || > + command == LOGICAL_REP_MSG_STREAM_PREPARE) > + memcpy(msg.m_gid, prepare_data->gid, GIDSIZE); else if (command == > + LOGICAL_REP_MSG_COMMIT_PREPARED) > + memcpy(msg.m_gid, commit_data->gid, GIDSIZE); else /* rollback > + prepared */ > + memcpy(msg.m_gid, rollback_data->gid, GIDSIZE); > AFTER: > + /* get the gid for this two phase operation */ if (command == > + LOGICAL_REP_MSG_PREPARE || > + command == LOGICAL_REP_MSG_STREAM_PREPARE) > + strlcpy(msg.m_gid, prepare_data->gid, sizeof(msg.m_gid)); else if > + (command == LOGICAL_REP_MSG_COMMIT_PREPARED) > + strlcpy(msg.m_gid, commit_data->gid, sizeof(msg.m_gid)); else /* > + rollback prepared */ > + strlcpy(msg.m_gid, rollback_data->gid, sizeof(msg.m_gid)); Fixed. > > BEFORE: > + strlcpy(prepared_txn->gid, msg->m_gid, strlen(msg->m_gid) + 1); > AFTER: > + strlcpy(prepared_txn->gid, msg->m_gid, sizeof(prepared_txn->gid)); > > BEFORE: > + memcpy(key.gid, msg->m_gid, strlen(msg->m_gid)); > AFTER: > + strlcpy(key.gid, msg->m_gid, sizeof(key.gid)); > > BEFORE: > + memcpy(key.gid, gid, strlen(gid)); > AFTER: > + strlcpy(key.gid, gid, sizeof(key.gid)); Fixed. > (10) src/backend/replication/logical/worker.c > Some suggested rewording: > > BEFORE: > + * size of streaming transaction resources because it have used the > AFTER: > + * size of streaming transaction resources because it has used the Fixed. > BEFORE: > + * tradeoff should not be good. Also, add multiple values > + * at once in order to reduce the number of this function call. > AFTER: > + * tradeoff would not be good. Also, add multiple values > + * at once in order to reduce the number of calls to this function. Fixed. > (11) update_apply_change_size() > Shouldn't this function be declared static? Fixed. > (12) stream_write_change() > > + streamed_entry->xact_size = streamed_entry->xact_size + total_len; > /* update */ > > could be simply written as: > > + streamed_entry->xact_size += total_len; /* update */ Fixed. Lastly, I removed one unnecessary test that checked publisher's stats in the TAP tests. Also I introduced ApplyTxnExtraData structure to remove void* argument of update_apply_change_size that might worsen the readability of codes in the previous version. Best Regards, Takamichi Osumi Attachments: [application/octet-stream] extend_xact_stats_of_subscription_worker_v8.patch (59.9K, ../../TYCPR01MB8373DC922E549FF3F031E9DAED8E9@TYCPR01MB8373.jpnprd01.prod.outlook.com/2-extend_xact_stats_of_subscription_worker_v8.patch) download | inline diff: From e6de87be6af810114d74b23e7d5a160499838157 Mon Sep 17 00:00:00 2001 From: Osumi Takamichi <[email protected]> Date: Fri, 5 Nov 2021 06:57:46 +0000 Subject: [PATCH v8] Extend pg_stat_subscription_workers to include general transaction statistics Categorize transactions of logical replication subscriber into three types (xact_commit, xact_error, xact_abort) and introduce cumulative columns of those numbers and amounts of consumed data during message apply respectively. One scenario to utilize those columns is to suppress unnecessary network bandwidth, when streaming transaction aborted more than expected is observed by the column of xact_abort_count or the column of its bytes. The calculation of consumed resources by subscriber is computed based on the data structure for message apply, which is different from that of publisher's decoding processing. Stats of prepared transaction becomes persistent to conclude the appropriate category for the transaction at either commit prepared or rollback prepared time. Also, streaming transactions running in parallel on publisher can cause data blocks in unpredictable order. Managing each streaming tranaction by xid makes it possible to handle stream abort, stream commit and stream prepare properly. Author: Takamichi Osumi Discussed & Reviewed-by: Amit Kapila, Masahiko Sawada, Hou Zhijie, Greg Nancarrow, Vignesh C Discussion: https://www.postgresql.org/message-id/OSBPR01MB48887CA8F40C8D984A6DC00CED199%40OSBPR01MB4888.jpnprd01.prod.outlook.com --- doc/src/sgml/monitoring.sgml | 94 +++++-- src/backend/catalog/system_views.sql | 16 +- src/backend/executor/execPartition.c | 10 + src/backend/postmaster/pgstat.c | 298 ++++++++++++++++++++++ src/backend/replication/logical/tablesync.c | 6 + src/backend/replication/logical/worker.c | 304 +++++++++++++++++++++++ src/backend/utils/adt/pgstatfuncs.c | 46 +++- src/include/catalog/pg_proc.dat | 6 +- src/include/executor/execPartition.h | 1 + src/include/pgstat.h | 91 +++++++ src/include/replication/logicalworker.h | 5 + src/test/regress/expected/rules.out | 18 +- src/test/subscription/t/026_error_report.pl | 8 +- src/test/subscription/t/027_worker_xact_stats.pl | 172 +++++++++++++ 14 files changed, 1030 insertions(+), 45 deletions(-) create mode 100644 src/test/subscription/t/027_worker_xact_stats.pl diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml index 094c723..4e9a68c 100644 --- a/doc/src/sgml/monitoring.sgml +++ b/doc/src/sgml/monitoring.sgml @@ -629,7 +629,7 @@ postgres 27093 0.0 0.0 30096 2752 ? Ss 11:34 0:00 postgres: ser <row> <entry><structname>pg_stat_subscription_workers</structname><indexterm><primary>pg_stat_subscription_workers</primary></indexterm></entry> - <entry>At least one row per subscription, showing about errors that + <entry>At least one row per subscription, showing transaction statistics and information about errors that occurred on subscription. See <link linkend="monitoring-pg-stat-subscription-workers"> <structname>pg_stat_subscription_workers</structname></link> for details. @@ -3052,9 +3052,9 @@ SELECT pid, wait_event_type, wait_event FROM pg_stat_activity WHERE wait_event i <para> The <structname>pg_stat_subscription_workers</structname> view will contain - one row per subscription error reported by workers applying logical - replication changes and workers handling the initial data copy of the - subscribed tables. + one row per subscription, showing corresponding transaction statistics and + information about the last error reported by workers applying logical replication + changes or by workers handling the initial data copy of the subscribed tables. </para> <table id="pg-stat-subscription-workers" xreflabel="pg_stat_subscription_workers"> @@ -3102,20 +3102,86 @@ SELECT pid, wait_event_type, wait_event FROM pg_stat_activity WHERE wait_event i <row> <entry role="catalog_table_entry"><para role="column_definition"> - <structfield>relid</structfield> <type>oid</type> + <structfield>xact_commit_count</structfield> <type>bigint</type> + </para> + <para> + Number of transactions successfully applied in this subscription. + COMMIT, COMMIT of streaming transaction and COMMIT PREPARED increments + this counter. + </para></entry> + </row> + + <row> + <entry role="catalog_table_entry"><para role="column_definition"> + <structfield>xact_commit_bytes</structfield> <type>bigint</type> + </para> + <para> + Amount of data (in bytes) successfully applied in this subscription, + across <literal>xact_commit_count</literal> transactions. + </para></entry> + </row> + + <row> + <entry role="catalog_table_entry"><para role="column_definition"> + <structfield>xact_error_count</structfield> <type>bigint</type> + </para> + <para> + Number of transactions that failed to be applied by the table + sync worker or main apply worker in this subscription. + </para></entry> + </row> + + <row> + <entry role="catalog_table_entry"><para role="column_definition"> + <structfield>xact_error_bytes</structfield> <type>bigint</type> + </para> + <para> + Amount of data (in bytes) unsuccessfully applied in this subscription, + across <literal>xact_error_count</literal> transactions. + </para></entry> + </row> + + <row> + <entry role="catalog_table_entry"><para role="column_definition"> + <structfield>xact_abort_count</structfield> <type>bigint</type> + </para> + <para> + Number of transactions aborted in this subscription. + Rollback of the prepared transaction and abort of streaming transaction + increments this counter. + </para></entry> + </row> + + <row> + <entry role="catalog_table_entry"><para role="column_definition"> + <structfield>xact_abort_bytes</structfield> <type>bigint</type> + </para> + <para> + Amount of data (in bytes) aborted in this subscription, + across <literal>xact_abort_count</literal> transactions. + In order to suppress unnecessary consumed network bandwidth, + increase <literal>logical_decoding_work_mem</literal> on the publisher + so that it exceeds the size of the whole streamed transaction, if + an unexpected amount of streamed transactions are aborted. + </para></entry> + </row> + + <row> + <entry role="catalog_table_entry"><para role="column_definition"> + <structfield>last_error_relid</structfield> <type>oid</type> </para> <para> OID of the relation that the worker was processing when the - error occurred + last error occurred </para></entry> </row> <row> <entry role="catalog_table_entry"><para role="column_definition"> - <structfield>command</structfield> <type>text</type> + <structfield>last_error_command</structfield> <type>text</type> </para> <para> - Name of command being applied when the error occurred. This field + Name of last command being applied when the error occurred. This field is always NULL if the error was reported during the initial data copy. </para></entry> @@ -3123,10 +3189,10 @@ SELECT pid, wait_event_type, wait_event FROM pg_stat_activity WHERE wait_event i <row> <entry role="catalog_table_entry"><para role="column_definition"> - <structfield>xid</structfield> <type>xid</type> + <structfield>last_error_xid</structfield> <type>xid</type> </para> <para> - Transaction ID of the publisher node being applied when the error + Transaction ID of the publisher node being applied when the last error occurred. This field is always NULL if the error was reported during the initial data copy. </para></entry> @@ -3134,19 +3200,19 @@ SELECT pid, wait_event_type, wait_event FROM pg_stat_activity WHERE wait_event i <row> <entry role="catalog_table_entry"><para role="column_definition"> - <structfield>error_count</structfield> <type>uint8</type> + <structfield>last_error_count</structfield> <type>uint8</type> </para> <para> - Number of consecutive times the error occurred + Number of consecutive times the last error occurred </para></entry> </row> <row> <entry role="catalog_table_entry"><para role="column_definition"> - <structfield>error_message</structfield> <type>text</type> + <structfield>last_error_message</structfield> <type>text</type> </para> <para> - The error message + The last error message </para></entry> </row> diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index a2ee00c..186b888 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1267,11 +1267,17 @@ CREATE VIEW pg_stat_subscription_workers AS w.subid, s.subname, w.subrelid, - w.relid, - w.command, - w.xid, - w.error_count, - w.error_message, + w.xact_commit_count, + w.xact_commit_bytes, + w.xact_error_count, + w.xact_error_bytes, + w.xact_abort_count, + w.xact_abort_bytes, + w.last_error_relid, + w.last_error_command, + w.last_error_xid, + w.last_error_count, + w.last_error_message, w.last_error_time, w.stats_reset FROM (SELECT diff --git a/src/backend/executor/execPartition.c b/src/backend/executor/execPartition.c index 5c723bc..20184c6 100644 --- a/src/backend/executor/execPartition.c +++ b/src/backend/executor/execPartition.c @@ -192,6 +192,16 @@ static void find_matching_subplans_recurse(PartitionPruningData *prunedata, bool initial_prune, Bitmapset **validsubplans); +/* + * PartitionTupleRoutingSize - exported to calculate total data size + * of logical replication mesage apply, because this is one of the + * ApplyExecutionData struct members. + */ +size_t +PartitionTupleRoutingSize(void) +{ + return sizeof(PartitionTupleRouting); +} /* * ExecSetupPartitionTupleRouting - sets up information needed during diff --git a/src/backend/postmaster/pgstat.c b/src/backend/postmaster/pgstat.c index b7883ec..9edbe47 100644 --- a/src/backend/postmaster/pgstat.c +++ b/src/backend/postmaster/pgstat.c @@ -54,6 +54,7 @@ #include "postmaster/fork_process.h" #include "postmaster/interrupt.h" #include "postmaster/postmaster.h" +#include "replication/logicalworker.h" #include "replication/slot.h" #include "replication/walsender.h" #include "storage/backendid.h" @@ -288,6 +289,15 @@ static HTAB *replSlotStatHash = NULL; static HTAB *subWorkerStatHash = NULL; /* + * Stats of prepared transactions should be displayed + * at either commit prepared or rollback prepared time, even when it's + * after the server restart. We have the apply worker send those statistics + * to the stats collector at prepare time and the startup process restore + * those at restart if necessary. + */ +static HTAB *subWorkerPreparedXactSizeHash = NULL; + +/* * List of OIDs of databases we need to write out. If an entry is InvalidOid, * it means to write only the shared-catalog stats ("DB 0"); otherwise, we * will write both that DB's data and the shared stats. @@ -338,6 +348,9 @@ static void pgstat_reset_replslot(PgStat_StatReplSlotEntry *slotstats, Timestamp static PgStat_StatSubWorkerEntry *pgstat_get_subworker_entry(Oid subid, Oid subrelid, bool create); +static PgStat_StatSubWorkerPreparedXactSize *pgstat_get_subworker_prepared_txn(Oid subid, + char *gid, bool create); + static void pgstat_reset_subworker_entry(PgStat_StatSubWorkerEntry *wentry, TimestampTz ts); static void pgstat_vacuum_subworker_stats(void); static void pgstat_send_subscription_purge(PgStat_MsgSubscriptionPurge *msg); @@ -388,6 +401,9 @@ static void pgstat_recv_tempfile(PgStat_MsgTempFile *msg, int len); static void pgstat_recv_subscription_purge(PgStat_MsgSubscriptionPurge *msg, int len); static void pgstat_recv_subworker_error(PgStat_MsgSubWorkerError *msg, int len); static void pgstat_recv_subworker_purge(PgStat_MsgSubWorkerPurge *msg, int len); +static void pgstat_recv_subworker_xact_end(PgStat_MsgSubWorkerXactEnd *msg, int len); +static void pgstat_recv_subworker_twophase_xact(PgStat_MsgSubWorkerTwophaseXact *msg, int len); + /* ------------------------------------------------------------ * Public functions called from postmaster follow @@ -2111,6 +2127,72 @@ pgstat_report_replslot_drop(const char *slotname) } /* ---------- + * pgstat_report_subworker_xact_end() - + * + * Tell the collector that worker transaction has successfully completed. + * ---------- + */ +void +pgstat_report_subworker_xact_end(Oid subid, Oid subrel, + LogicalRepMsgType command, PgStat_Counter xact_size) +{ + PgStat_MsgSubWorkerXactEnd msg; + + Assert(command == 0 /* table sync worker */ || + command == LOGICAL_REP_MSG_COMMIT || + command == LOGICAL_REP_MSG_STREAM_ABORT || + command == LOGICAL_REP_MSG_STREAM_COMMIT); + + pgstat_setheader(&msg.m_hdr, PGSTAT_MTYPE_SUBWORKERXACTEND); + msg.m_subid = subid; + msg.m_subrelid = subrel; + msg.m_command = command; + msg.m_xact_bytes = xact_size; + pgstat_send(&msg, sizeof(PgStat_MsgSubWorkerXactEnd)); + + reset_apply_error_context_xact_size(); +} + +/* ---------- + * pgstat_report_subworker_twophase_xact() - + * + * Tell the collector that worker transaction has done 2PC related operation. + * ---------- + */ +void +pgstat_report_subworker_twophase_xact(Oid subid, LogicalRepMsgType command, + PgStat_Counter xact_size, + LogicalRepPreparedTxnData *prepare_data, + LogicalRepCommitPreparedTxnData *commit_data, + LogicalRepRollbackPreparedTxnData *rollback_data) +{ + PgStat_MsgSubWorkerTwophaseXact msg; + + Assert(command == LOGICAL_REP_MSG_PREPARE || + command == LOGICAL_REP_MSG_STREAM_PREPARE || + command == LOGICAL_REP_MSG_COMMIT_PREPARED || + command == LOGICAL_REP_MSG_ROLLBACK_PREPARED); + + /* setup the message */ + pgstat_setheader(&msg.m_hdr, PGSTAT_MTYPE_SUBWORKERTWOPHASEXACT); + msg.m_subid = subid; + msg.m_command = command; + + /* get the gid for this two phase operation */ + if (command == LOGICAL_REP_MSG_PREPARE || + command == LOGICAL_REP_MSG_STREAM_PREPARE) + strlcpy(msg.m_gid, prepare_data->gid, sizeof(msg.m_gid)); + else if (command == LOGICAL_REP_MSG_COMMIT_PREPARED) + strlcpy(msg.m_gid, commit_data->gid, sizeof(msg.m_gid)); + else /* rollback prepared */ + strlcpy(msg.m_gid, rollback_data->gid, sizeof(msg.m_gid)); + + msg.m_xact_bytes = xact_size; + pgstat_send(&msg, sizeof(PgStat_MsgSubWorkerTwophaseXact)); + reset_apply_error_context_xact_size(); +} + +/* ---------- * pgstat_report_subworker_error() - * * Tell the collector about the subscription worker error. @@ -2130,6 +2212,7 @@ pgstat_report_subworker_error(Oid subid, Oid subrelid, Oid relid, pgstat_setheader(&msg.m_hdr, PGSTAT_MTYPE_SUBWORKERERROR); msg.m_subid = subid; msg.m_subrelid = subrelid; + msg.m_xact_error_bytes = get_apply_error_context_xact_size(); msg.m_dbid = MyDatabaseId; msg.m_relid = relid; msg.m_command = command; @@ -2138,6 +2221,8 @@ pgstat_report_subworker_error(Oid subid, Oid subrelid, Oid relid, strlcpy(msg.m_message, errmsg, PGSTAT_SUBWORKERERROR_MSGLEN); pgstat_send(&msg, len); + + reset_apply_error_context_xact_size(); } /* ---------- @@ -3906,6 +3991,14 @@ PgstatCollectorMain(int argc, char *argv[]) pgstat_recv_subworker_purge(&msg.msg_subworkerpurge, len); break; + case PGSTAT_MTYPE_SUBWORKERXACTEND: + pgstat_recv_subworker_xact_end(&msg.msg_subworkerxactend, len); + break; + + case PGSTAT_MTYPE_SUBWORKERTWOPHASEXACT: + pgstat_recv_subworker_twophase_xact(&msg.msg_subworkertwophasexact, len); + break; + default: break; } @@ -4223,6 +4316,22 @@ pgstat_write_statsfiles(bool permanent, bool allDbs) } /* + * Write subscription worker's prepared transaction struct + */ + if (subWorkerPreparedXactSizeHash) + { + PgStat_StatSubWorkerPreparedXactSize *prepared_size; + + hash_seq_init(&hstat, subWorkerPreparedXactSizeHash); + while((prepared_size = (PgStat_StatSubWorkerPreparedXactSize *) hash_seq_search(&hstat)) != NULL) + { + fputc('P', fpout); + rc = fwrite(prepared_size, sizeof(PgStat_StatSubWorkerPreparedXactSize), 1, fpout); + (void) rc; /* we'll check for error with ferror */ + } + } + + /* * No more output to be done. Close the temp file and replace the old * pgstat.stat with it. The ferror() check replaces testing for error * after each individual fputc or fwrite above. @@ -4725,6 +4834,40 @@ pgstat_read_statsfiles(Oid onlydb, bool permanent, bool deep) break; } + case 'P': + { + PgStat_StatSubWorkerPreparedXactSize buff; + PgStat_StatSubWorkerPreparedXactSize *prepared_xact_size; + + if (fread(&buff, 1, sizeof(PgStat_StatSubWorkerPreparedXactSize), + fpin) != sizeof(PgStat_StatSubWorkerPreparedXactSize)) + { + ereport(pgStatRunningInCollector ? LOG : WARNING, + (errmsg("corrupted statistics file \"%s\"", + statfile))); + goto done; + } + + if (subWorkerPreparedXactSizeHash == NULL) + { + HASHCTL hash_ctl; + + hash_ctl.keysize = sizeof(PgStat_StatSubWorkerPreparedXact); + hash_ctl.entrysize = sizeof(PgStat_StatSubWorkerPreparedXactSize); + hash_ctl.hcxt = pgStatLocalContext; + subWorkerPreparedXactSizeHash = hash_create("Subscription worker stats of prepared txn", + PGSTAT_SUBWORKER_HASH_SIZE, + &hash_ctl, + HASH_ELEM | HASH_STRINGS | HASH_CONTEXT); + } + + prepared_xact_size = + (PgStat_StatSubWorkerPreparedXactSize *) hash_search(subWorkerPreparedXactSizeHash, + (void *) &buff.key, + HASH_ENTER, NULL); + memcpy(prepared_xact_size, &buff, sizeof(PgStat_StatSubWorkerPreparedXactSize)); + break; + } case 'E': goto done; @@ -5290,6 +5433,7 @@ pgstat_clear_snapshot(void) pgStatDBHash = NULL; replSlotStatHash = NULL; subWorkerStatHash = NULL; + subWorkerPreparedXactSizeHash = NULL; /* * Historically the backend_status.c facilities lived in this file, and @@ -6287,6 +6431,105 @@ pgstat_recv_subscription_purge(PgStat_MsgSubscriptionPurge *msg, int len) } /* ---------- + * pgstat_recv_subworker_xact_end() - + * + * Process a SUBWORKERXACTEND message. + * ---------- + */ +static void +pgstat_recv_subworker_xact_end(PgStat_MsgSubWorkerXactEnd *msg, int len) +{ + PgStat_StatSubWorkerEntry *wentry; + + wentry = pgstat_get_subworker_entry(msg->m_subid, msg->m_subrelid, true); + Assert(wentry); + + /* table sync worker */ + if (msg->m_command == 0) + wentry->xact_commit_count++; + else + { + /* apply worker */ + switch(msg->m_command) + { + case LOGICAL_REP_MSG_COMMIT: + case LOGICAL_REP_MSG_STREAM_COMMIT: + wentry->xact_commit_count++; + wentry->xact_commit_bytes += msg->m_xact_bytes; + break; + case LOGICAL_REP_MSG_STREAM_ABORT: + wentry->xact_abort_count++; + wentry->xact_abort_bytes += msg->m_xact_bytes; + break; + default: + elog(ERROR, "unexpected logical message type as normal apply end"); + break; + } + } +} + +/* ---------- + * pgstat_recv_subworker_twophase_xact() - + * + * Process a SUBWORKERTWOPHASEXACT message. + * ---------- + */ +static void +pgstat_recv_subworker_twophase_xact(PgStat_MsgSubWorkerTwophaseXact *msg, int len) +{ + PgStat_StatSubWorkerPreparedXactSize *prepared_txn; + PgStat_StatSubWorkerEntry *wentry; + PgStat_StatSubWorkerPreparedXact key; + + prepared_txn = pgstat_get_subworker_prepared_txn(msg->m_subid, msg->m_gid, true); + Assert(prepared_txn); + switch(msg->m_command) + { + case LOGICAL_REP_MSG_PREPARE: + case LOGICAL_REP_MSG_STREAM_PREPARE: + /* + * Make each size of prepared transaction persistent + * so that we can update stats over the server restart + * and make prepared stats updated when commit prepared + * or rollback prepared arrives. + */ + prepared_txn->subid = msg->m_subid; + strlcpy(prepared_txn->gid, msg->m_gid, sizeof(prepared_txn->gid)); + prepared_txn->xact_size = msg->m_xact_bytes; + break; + + case LOGICAL_REP_MSG_COMMIT_PREPARED: + case LOGICAL_REP_MSG_ROLLBACK_PREPARED: + /* Update exported xact stats now */ + wentry = pgstat_get_subworker_entry(msg->m_subid, + InvalidOid /* apply worker */, + true); + Assert(wentry); + if (msg->m_command == LOGICAL_REP_MSG_COMMIT_PREPARED) + { + wentry->xact_commit_count++; + wentry->xact_commit_bytes += prepared_txn->xact_size; + } + else + { + wentry->xact_abort_count++; + wentry->xact_abort_bytes += prepared_txn->xact_size; + } + + /* Clean up this gid from transaction size hash */ + key.subid = prepared_txn->subid; + strlcpy(key.gid, msg->m_gid, strlen(key.gid)); + (void) hash_search(subWorkerPreparedXactSizeHash, + (void *) &key, HASH_REMOVE, NULL); + break; + + default: + elog(ERROR, "unexpected logical message type as prepare transaction"); + break; + } +} + +/* ---------- * pgstat_recv_subworker_error() - * * Process a SUBWORKERERROR message. @@ -6301,6 +6544,10 @@ pgstat_recv_subworker_error(PgStat_MsgSubWorkerError *msg, int len) wentry = pgstat_get_subworker_entry(msg->m_subid, msg->m_subrelid, true); Assert(wentry); + /* general transaction stats for error */ + wentry->xact_error_count++; + wentry->xact_error_bytes += msg->m_xact_error_bytes; + /* * Update only the counter and timestamp if we received the same error * again @@ -6520,6 +6767,51 @@ pgstat_get_subworker_entry(Oid subid, Oid subrelid, bool create) } /* ---------- + * pgstat_get_subworker_prepared_txn + * + * Return subscription worker entry with the given subscription OID and + * gid. + * ---------- + */ +static PgStat_StatSubWorkerPreparedXactSize* +pgstat_get_subworker_prepared_txn(Oid subid, char *gid, bool create) +{ + PgStat_StatSubWorkerPreparedXact key; + PgStat_StatSubWorkerPreparedXactSize *prepared_txn_size; + HASHACTION action; + bool found; + + if (subWorkerPreparedXactSizeHash == NULL) + { + HASHCTL hash_ctl; + + hash_ctl.keysize = sizeof(PgStat_StatSubWorkerPreparedXact); + hash_ctl.entrysize = sizeof(PgStat_StatSubWorkerPreparedXactSize); + hash_ctl.hcxt = pgStatLocalContext; + subWorkerPreparedXactSizeHash = hash_create("Subscription worker stats of prepared txn", + PGSTAT_SUBWORKER_HASH_SIZE, + &hash_ctl, + HASH_ELEM | HASH_STRINGS); + } + + key.subid = subid; + memcpy(key.gid, gid, strlen(key.gid)); + action = (create ? HASH_ENTER : HASH_FIND); + prepared_txn_size = (PgStat_StatSubWorkerPreparedXactSize *) hash_search(subWorkerPreparedXactSizeHash, + (void *) &key, + action, &found); + + if (create && !found) + { + prepared_txn_size->subid = 0; + prepared_txn_size->gid[0] = '\0'; + prepared_txn_size->xact_size = 0; + } + + return prepared_txn_size; +} + +/* ---------- * pgstat_reset_subworker_entry * * Reset the given subscription worker statistics. @@ -6528,6 +6820,12 @@ pgstat_get_subworker_entry(Oid subid, Oid subrelid, bool create) static void pgstat_reset_subworker_entry(PgStat_StatSubWorkerEntry *wentry, TimestampTz ts) { + wentry->xact_commit_count = 0; + wentry->xact_commit_bytes = 0; + wentry->xact_error_count = 0; + wentry->xact_error_bytes = 0; + wentry->xact_abort_count = 0; + wentry->xact_abort_bytes = 0; wentry->dbid = InvalidOid; wentry->relid = InvalidOid; wentry->command = 0; diff --git a/src/backend/replication/logical/tablesync.c b/src/backend/replication/logical/tablesync.c index 9b6d057..0d04ea5 100644 --- a/src/backend/replication/logical/tablesync.c +++ b/src/backend/replication/logical/tablesync.c @@ -1162,6 +1162,12 @@ copy_table_done: MyLogicalRepWorker->relstate_lsn = *origin_startpos; SpinLockRelease(&MyLogicalRepWorker->relmutex); + /* Report the success of table sync. */ + pgstat_report_subworker_xact_end(MyLogicalRepWorker->subid, + MyLogicalRepWorker->relid, + 0 /* no logical message type */, + 0 /* xact size */); + /* * Finally, wait until the main apply worker tells us to catch up and then * return to let LogicalRepApplyLoop do it. diff --git a/src/backend/replication/logical/worker.c b/src/backend/replication/logical/worker.c index 3a40684..04d4236 100644 --- a/src/backend/replication/logical/worker.c +++ b/src/backend/replication/logical/worker.c @@ -221,21 +221,65 @@ typedef struct ApplyErrorCallbackArg LogicalRepMsgType command; /* 0 if invalid */ LogicalRepRelMapEntry *rel; + /* + * Store data size of this transaction. + * + * The byte size of transaction on the publisher is calculated + * by ReorderBufferChangeSize() based on the ReorderBufferChange + * structure. But on the subscriber, consumed resources are + * not same as the publisher's decoding processsing and required + * to be computed in different way. Therefore, the exact same byte + * size is not restored on the subscriber usually. + * + * Data size of streaming transactions is managed by streamingXactSize + * for flexible data blocks handling. + */ + PgStat_Counter bytes; + /* Remote node information */ int remote_attnum; /* -1 if invalid */ TransactionId remote_xid; TimestampTz ts; /* commit, rollback, or prepare timestamp */ } ApplyErrorCallbackArg; +/* Struct to indicate extra consumption of transaction size */ +typedef union ApplyTxnExtraData +{ + LogicalRepRelMapEntry *relmapentry; + LogicalRepRelation *reprelation; + int *stream_write_len; +} ApplyTxnExtraData; + static ApplyErrorCallbackArg apply_error_callback_arg = { .command = 0, .rel = NULL, + .bytes = 0, .remote_attnum = -1, .remote_xid = InvalidTransactionId, .ts = 0, }; +/* + * Two or more streaming transactions in parallel on the publisher + * generate unexpected order of partial txn data demarcated stream start + * and stream stop to the subscriber, since whenever its size of one of + * the txns reaches the publisher's logical_decoding_work_mem, + * the part (and in the end, last remaining changes) is streamed. + * This creates mixed blocks of streaming data while + * there's possibility some are successfully committed but others are + * not by stream abort. Therefore, to track correct byte size + * it's necessary to trace each streaming transaction by making pair + * of xid and transaction size. + */ +#define PARALLEL_STREAMING_XACTS 32 +typedef struct XactSizeEntry +{ + TransactionId key; + PgStat_Counter xact_size; +} XactSizeEntry; +static HTAB *streamingXactSize = NULL; + static MemoryContext ApplyMessageContext = NULL; MemoryContext ApplyContext = NULL; @@ -304,6 +348,9 @@ static void maybe_reread_subscription(void); /* prototype needed because of stream_commit */ static void apply_dispatch(StringInfo s); +static void update_apply_change_size(LogicalRepMsgType action, + ApplyTxnExtraData *extra_data); + static void apply_handle_commit_internal(LogicalRepCommitData *commit_data); static void apply_handle_insert_internal(ApplyExecutionData *edata, ResultRelInfo *relinfo, @@ -818,6 +865,11 @@ apply_handle_commit(StringInfo s) /* Process any tables that are being synchronized in parallel. */ process_syncing_tables(commit_data.end_lsn); + pgstat_report_subworker_xact_end(MyLogicalRepWorker->subid, + InvalidOid, + LOGICAL_REP_MSG_COMMIT, + get_apply_error_context_xact_size()); + pgstat_report_activity(STATE_IDLE, NULL); reset_apply_error_context_info(); } @@ -922,6 +974,11 @@ apply_handle_prepare(StringInfo s) /* Process any tables that are being synchronized in parallel. */ process_syncing_tables(prepare_data.end_lsn); + pgstat_report_subworker_twophase_xact(MyLogicalRepWorker->subid, + LOGICAL_REP_MSG_PREPARE, + get_apply_error_context_xact_size(), + &prepare_data, NULL, NULL); + pgstat_report_activity(STATE_IDLE, NULL); reset_apply_error_context_info(); } @@ -963,6 +1020,13 @@ apply_handle_commit_prepared(StringInfo s) /* Process any tables that are being synchronized in parallel. */ process_syncing_tables(prepare_data.end_lsn); + update_apply_change_size(LOGICAL_REP_MSG_COMMIT_PREPARED, NULL); + + pgstat_report_subworker_twophase_xact(MyLogicalRepWorker->subid, + LOGICAL_REP_MSG_COMMIT_PREPARED, + get_apply_error_context_xact_size(), + NULL, &prepare_data, NULL); + pgstat_report_activity(STATE_IDLE, NULL); reset_apply_error_context_info(); } @@ -1014,6 +1078,12 @@ apply_handle_rollback_prepared(StringInfo s) /* Process any tables that are being synchronized in parallel. */ process_syncing_tables(rollback_data.rollback_end_lsn); + /* send rollback prepared message for this gid */ + pgstat_report_subworker_twophase_xact(MyLogicalRepWorker->subid, + LOGICAL_REP_MSG_ROLLBACK_PREPARED, + get_apply_error_context_xact_size(), + NULL, NULL, &rollback_data); + pgstat_report_activity(STATE_IDLE, NULL); reset_apply_error_context_info(); } @@ -1029,6 +1099,7 @@ static void apply_handle_stream_prepare(StringInfo s) { LogicalRepPreparedTxnData prepare_data; + XactSizeEntry *streamed_entry; if (in_streamed_transaction) ereport(ERROR, @@ -1066,6 +1137,19 @@ apply_handle_stream_prepare(StringInfo s) /* Process any tables that are being synchronized in parallel. */ process_syncing_tables(prepare_data.end_lsn); + /* + * Report the prepared streaming xact size to the stats collector + * in a prepared xact manner to make it survive over the restart. + */ + streamed_entry = hash_search(streamingXactSize, + (void *) &prepare_data.xid, + HASH_FIND, NULL); + Assert(streamed_entry); + pgstat_report_subworker_twophase_xact(MyLogicalRepWorker->subid, + LOGICAL_REP_MSG_STREAM_PREPARE, + streamed_entry->xact_size, + &prepare_data, NULL, NULL); + pgstat_report_activity(STATE_IDLE, NULL); reset_apply_error_context_info(); @@ -1143,6 +1227,8 @@ apply_handle_stream_start(StringInfo s) MyLogicalRepWorker->stream_fileset = palloc(sizeof(FileSet)); FileSetInit(MyLogicalRepWorker->stream_fileset); + update_apply_change_size(LOGICAL_REP_MSG_STREAM_START, NULL); + MemoryContextSwitchTo(oldctx); } @@ -1193,12 +1279,17 @@ apply_handle_stream_stop(StringInfo s) /* * Handle STREAM abort message. + * + * Currently, abort of streaming subtransaction does not affect + * size of streaming transaction resources because it has used the + * resources anyway. */ static void apply_handle_stream_abort(StringInfo s) { TransactionId xid; TransactionId subxid; + XactSizeEntry *streamed_entry; if (in_streamed_transaction) ereport(ERROR, @@ -1213,8 +1304,36 @@ apply_handle_stream_abort(StringInfo s) */ if (xid == subxid) { + bool found = false; set_apply_error_context_xact(xid, 0); stream_cleanup_files(MyLogicalRepWorker->subid, xid); + + /* + * We've completed to handle stream abort without issue, so + * get ready to report the transaction stats via normal + * termination route instead of the apply error route. + */ + streamed_entry = hash_search(streamingXactSize, + (void *) &xid, + HASH_FIND, &found); + /* + * It's possible that we get stream abort + * earlier than any call of write_stream_change that + * creates one hash entry for this xid. In this case, + * to find a entry with this xid fails. So just check + * if we've found it. Only when we confirm some writes + * by write_stream_change, report the stream_abort. + */ + if (found) + { + pgstat_report_subworker_xact_end(MyLogicalRepWorker->subid, + InvalidOid, + LOGICAL_REP_MSG_STREAM_ABORT, + streamed_entry->xact_size); + (void) hash_search(streamingXactSize, + (void *) &xid, + HASH_REMOVE, NULL); + } } else { @@ -1417,6 +1536,7 @@ apply_handle_stream_commit(StringInfo s) { TransactionId xid; LogicalRepCommitData commit_data; + XactSizeEntry *streamed_entry; if (in_streamed_transaction) ereport(ERROR, @@ -1438,6 +1558,19 @@ apply_handle_stream_commit(StringInfo s) /* Process any tables that are being synchronized in parallel. */ process_syncing_tables(commit_data.end_lsn); + /* Report and clean up the xid */ + streamed_entry = hash_search(streamingXactSize, + (void *) &xid, + HASH_FIND, NULL); + Assert(streamed_entry); + pgstat_report_subworker_xact_end(MyLogicalRepWorker->subid, + InvalidOid, + LOGICAL_REP_MSG_STREAM_COMMIT, + streamed_entry->xact_size); + (void) hash_search(streamingXactSize, + (void *) &xid, + HASH_REMOVE, NULL); + pgstat_report_activity(STATE_IDLE, NULL); reset_apply_error_context_info(); @@ -1541,6 +1674,7 @@ apply_handle_insert(StringInfo s) EState *estate; TupleTableSlot *remoteslot; MemoryContext oldctx; + ApplyTxnExtraData extra_size; if (handle_streamed_transaction(LOGICAL_REP_MSG_INSERT, s)) return; @@ -1576,6 +1710,10 @@ apply_handle_insert(StringInfo s) slot_fill_defaults(rel, estate, remoteslot); MemoryContextSwitchTo(oldctx); + /* Update transaction size */ + extra_size.relmapentry = rel; + update_apply_change_size(LOGICAL_REP_MSG_INSERT, &extra_size); + /* For a partitioned table, insert the tuple into a partition. */ if (rel->localrel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE) apply_handle_tuple_routing(edata, @@ -1667,6 +1805,7 @@ apply_handle_update(StringInfo s) TupleTableSlot *remoteslot; RangeTblEntry *target_rte; MemoryContext oldctx; + ApplyTxnExtraData extra_size; if (handle_streamed_transaction(LOGICAL_REP_MSG_UPDATE, s)) return; @@ -1733,6 +1872,10 @@ apply_handle_update(StringInfo s) has_oldtup ? &oldtup : &newtup); MemoryContextSwitchTo(oldctx); + /* Update transaction size */ + extra_size.relmapentry = rel; + update_apply_change_size(LOGICAL_REP_MSG_UPDATE, &extra_size); + /* For a partitioned table, apply update to correct partition. */ if (rel->localrel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE) apply_handle_tuple_routing(edata, @@ -1830,6 +1973,7 @@ apply_handle_delete(StringInfo s) EState *estate; TupleTableSlot *remoteslot; MemoryContext oldctx; + ApplyTxnExtraData extra_size; if (handle_streamed_transaction(LOGICAL_REP_MSG_DELETE, s)) return; @@ -1867,6 +2011,10 @@ apply_handle_delete(StringInfo s) slot_store_data(remoteslot, rel, &oldtup); MemoryContextSwitchTo(oldctx); + /* Update transaction size */ + extra_size.relmapentry = rel; + update_apply_change_size(LOGICAL_REP_MSG_DELETE, &extra_size); + /* For a partitioned table, apply delete to correct partition. */ if (rel->localrel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE) apply_handle_tuple_routing(edata, @@ -2418,6 +2566,111 @@ apply_dispatch(StringInfo s) } /* + * Subscriber side implementation equivalent to ReorderBufferChangeSize + * of the publisher. + * + * According to the logical replication message type, record major + * resource consumptions of this subscription for each message. + * At present, do not collect data from generic functions to keep + * code simplicity, since the implementation complexity versus benefit + * tradeoff should not be good. Also, add multiple values + * at once in order to reduce the number of calls to this function. + * + * 'extra_data' controls detail handling of data size calculation. + */ +static void +update_apply_change_size(LogicalRepMsgType action, ApplyTxnExtraData *extra_data) +{ + int64 size = 0; + + /* + * In streaming mode, stream_write_change is called + * instead of immediate apply. List up the messages types + * that can be caught by handle_streamed_transaction and + * treat the write length as the size of transaction so + * that we can export it as part of pg_stat_subscription_worker. + */ + if (in_streamed_transaction && + (action == LOGICAL_REP_MSG_INSERT || + action == LOGICAL_REP_MSG_UPDATE || + action == LOGICAL_REP_MSG_DELETE || + action == LOGICAL_REP_MSG_TRUNCATE || + action == LOGICAL_REP_MSG_RELATION || + action == LOGICAL_REP_MSG_TYPE)) + { + size += *extra_data->stream_write_len; + add_apply_error_context_xact_size(size); + return; + } + + switch (action) + { + /* No special memory consumption */ + case LOGICAL_REP_MSG_BEGIN: + case LOGICAL_REP_MSG_COMMIT: + case LOGICAL_REP_MSG_TRUNCATE: + case LOGICAL_REP_MSG_TYPE: + case LOGICAL_REP_MSG_ORIGIN: + case LOGICAL_REP_MSG_MESSAGE: + case LOGICAL_REP_MSG_STREAM_STOP: + case LOGICAL_REP_MSG_STREAM_ABORT: + case LOGICAL_REP_MSG_BEGIN_PREPARE: + break; + + case LOGICAL_REP_MSG_INSERT: + case LOGICAL_REP_MSG_UPDATE: + case LOGICAL_REP_MSG_DELETE: + Assert(extra_data != NULL); + + /* + * Compute size based on ApplyExecutionData. + * The size of LogicalRepRelMapEntry can be skipped because + * it is obtained from hash_search in logicalrep_rel_open. + */ + size += sizeof(ApplyExecutionData) + sizeof(EState) + + sizeof(ResultRelInfo) + sizeof(ResultRelInfo); + + /* + * Add some extra size if the target relation is partitioned. + * PartitionTupleRouting isn't exported. Therefore, call the + * function that returns its size instead. + */ + if (extra_data->relmapentry->localrel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE) + size += sizeof(ModifyTableState) + PartitionTupleRoutingSize(); + break; + + case LOGICAL_REP_MSG_RELATION: + Assert(extra_data != NULL); + + /* See logicalrep_read_attrs for the last two */ + size += sizeof(LogicalRepRelation) + + extra_data->reprelation->natts * sizeof(char *) + + extra_data->reprelation->natts * sizeof(Oid); + break; + + case LOGICAL_REP_MSG_STREAM_START: + size += sizeof(FileSet); + break; + + case LOGICAL_REP_MSG_PREPARE: + case LOGICAL_REP_MSG_COMMIT_PREPARED: + case LOGICAL_REP_MSG_ROLLBACK_PREPARED: + case LOGICAL_REP_MSG_STREAM_COMMIT: + case LOGICAL_REP_MSG_STREAM_PREPARE: + size += sizeof(FlushPosition); + break; + + default: + ereport(ERROR, + (errcode(ERRCODE_PROTOCOL_VIOLATION), + errmsg("invalid logical replication message type \"%c\"", action))); + } + + /* update the total size of consumption */ + add_apply_error_context_xact_size(size); +} + +/* * Figure out which write/flush positions to report to the walsender process. * * We can't simply report back the last LSN the walsender sent us because the @@ -3271,6 +3524,9 @@ static void stream_write_change(char action, StringInfo s) { int len; + int total_len; + bool found; + XactSizeEntry *streamed_entry; Assert(in_streamed_transaction); Assert(TransactionIdIsValid(stream_xid)); @@ -3289,6 +3545,16 @@ stream_write_change(char action, StringInfo s) len = (s->len - s->cursor); BufFileWrite(stream_fd, &s->data[s->cursor], len); + + /* update xact size by xid */ + total_len = (s->len - s->cursor) * 2 + sizeof(char) + sizeof(action); + streamed_entry = (XactSizeEntry *) hash_search(streamingXactSize, + (void *) &stream_xid, + HASH_ENTER, &found); + if (!found) + streamed_entry->xact_size = total_len; /* init */ + else + streamed_entry->xact_size += total_len; /* update */ } /* @@ -3426,6 +3692,23 @@ ApplyWorkerMain(Datum main_arg) elog(DEBUG1, "connecting to publisher using connection string \"%s\"", MySubscription->conninfo); + /* + * Initialize the apply worker's hash to manage bytes for + * streaming txns. + */ + if (!am_tablesync_worker() && MySubscription->stream) + { + HASHCTL hash_ctl; + + hash_ctl.keysize = sizeof(TransactionId); + hash_ctl.entrysize = sizeof(XactSizeEntry); + hash_ctl.hcxt = LogicalStreamingContext; + streamingXactSize = hash_create("xact size per streaming xid", + PARALLEL_STREAMING_XACTS, + &hash_ctl, + HASH_ELEM | HASH_BLOBS | HASH_CONTEXT); + } + if (am_tablesync_worker()) { char *syncslotname; @@ -3601,6 +3884,27 @@ ApplyWorkerMain(Datum main_arg) proc_exit(0); } +/* Exported so that stats collector can utilize this value */ +int64 +get_apply_error_context_xact_size(void) +{ + return apply_error_callback_arg.bytes; +} + +/* Add size to apply error bytes */ +void +add_apply_error_context_xact_size(int64 size) +{ + apply_error_callback_arg.bytes += size; +} + +/* Reset information of apply error callback */ +void +reset_apply_error_context_xact_size(void) +{ + apply_error_callback_arg.bytes = 0; +} + /* * Is current process a logical replication worker? */ diff --git a/src/backend/utils/adt/pgstatfuncs.c b/src/backend/utils/adt/pgstatfuncs.c index 2511df1..31aa46e 100644 --- a/src/backend/utils/adt/pgstatfuncs.c +++ b/src/backend/utils/adt/pgstatfuncs.c @@ -2405,7 +2405,7 @@ pg_stat_get_replication_slot(PG_FUNCTION_ARGS) Datum pg_stat_get_subscription_worker(PG_FUNCTION_ARGS) { -#define PG_STAT_GET_SUBSCRIPTION_WORKER_COLS 9 +#define PG_STAT_GET_SUBSCRIPTION_WORKER_COLS 15 Oid subid = PG_GETARG_OID(0); Oid subrelid; TupleDesc tupdesc; @@ -2425,19 +2425,31 @@ pg_stat_get_subscription_worker(PG_FUNCTION_ARGS) OIDOID, -1, 0); TupleDescInitEntry(tupdesc, (AttrNumber) 2, "subrelid", OIDOID, -1, 0); - TupleDescInitEntry(tupdesc, (AttrNumber) 3, "relid", + TupleDescInitEntry(tupdesc, (AttrNumber) 3, "xact_commit_count", + INT8OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) 4, "xact_commit_bytes", + INT8OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) 5, "xact_error_count", + INT8OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) 6, "xact_error_bytes", + INT8OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) 7, "xact_abort_count", + INT8OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) 8, "xact_abort_bytes", + INT8OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) 9, "last_error_relid", OIDOID, -1, 0); - TupleDescInitEntry(tupdesc, (AttrNumber) 4, "command", + TupleDescInitEntry(tupdesc, (AttrNumber) 10, "last_error_command", TEXTOID, -1, 0); - TupleDescInitEntry(tupdesc, (AttrNumber) 5, "xid", + TupleDescInitEntry(tupdesc, (AttrNumber) 11, "last_error_xid", XIDOID, -1, 0); - TupleDescInitEntry(tupdesc, (AttrNumber) 6, "error_count", + TupleDescInitEntry(tupdesc, (AttrNumber) 12, "last_error_count", INT8OID, -1, 0); - TupleDescInitEntry(tupdesc, (AttrNumber) 7, "error_message", + TupleDescInitEntry(tupdesc, (AttrNumber) 13, "last_error_message", TEXTOID, -1, 0); - TupleDescInitEntry(tupdesc, (AttrNumber) 8, "last_error_time", + TupleDescInitEntry(tupdesc, (AttrNumber) 14, "last_error_time", TIMESTAMPTZOID, -1, 0); - TupleDescInitEntry(tupdesc, (AttrNumber) 9, "stats_reset", + TupleDescInitEntry(tupdesc, (AttrNumber) 15, "stats_reset", TIMESTAMPTZOID, -1, 0); BlessTupleDesc(tupdesc); @@ -2462,28 +2474,36 @@ pg_stat_get_subscription_worker(PG_FUNCTION_ARGS) else nulls[i++] = true; - /* relid */ + /* transaction stats */ + values[i++] = Int64GetDatum(wentry->xact_commit_count); + values[i++] = Int64GetDatum(wentry->xact_commit_bytes); + values[i++] = Int64GetDatum(wentry->xact_error_count); + values[i++] = Int64GetDatum(wentry->xact_error_bytes); + values[i++] = Int64GetDatum(wentry->xact_abort_count); + values[i++] = Int64GetDatum(wentry->xact_abort_bytes); + + /* last_error_relid */ if (OidIsValid(wentry->relid)) values[i++] = ObjectIdGetDatum(wentry->relid); else nulls[i++] = true; - /* command */ + /* last_error_command */ if (wentry->command != 0) values[i++] = CStringGetTextDatum(logicalrep_message_type(wentry->command)); else nulls[i++] = true; - /* xid */ + /* last_error_xid */ if (TransactionIdIsValid(wentry->xid)) values[i++] = TransactionIdGetDatum(wentry->xid); else nulls[i++] = true; - /* error_count */ + /* last_error_count */ values[i++] = Int64GetDatum(wentry->error_count); - /* error_message */ + /* last_error_message */ values[i++] = CStringGetTextDatum(wentry->error_message); /* last_error_time */ diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index e6c7abb..ae372a5 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -5389,9 +5389,9 @@ proname => 'pg_stat_get_subscription_worker', prorows => '1', proisstrict => 'f', proretset => 't', provolatile => 's', proparallel => 'r', prorettype => 'record', proargtypes => 'oid oid', - proallargtypes => '{oid,oid,oid,oid,oid,text,xid,int8,text,timestamptz,timestamptz}', - proargmodes => '{i,i,o,o,o,o,o,o,o,o,o}', - proargnames => '{subid,subrelid,subid,subrelid,relid,command,xid,error_count,error_message,last_error_time,stats_reset}', + proallargtypes => '{oid,oid,oid,oid,int8,int8,int8,int8,int8,int8,oid,text,xid,int8,text,timestamptz,timestamptz}', + proargmodes => '{i,i,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o}', + proargnames => '{subid,subrelid,subid,subrelid,xact_commit_count,xact_commit_bytes,xact_error_count,xact_error_bytes,xact_abort_count,xact_abort_bytes,last_error_relid,last_error_command,last_error_xid,last_error_count,last_error_message,last_error_time,stats_reset}', prosrc => 'pg_stat_get_subscription_worker' }, { oid => '6118', descr => 'statistics: information about subscription', proname => 'pg_stat_get_subscription', prorows => '10', proisstrict => 'f', diff --git a/src/include/executor/execPartition.h b/src/include/executor/execPartition.h index 694e38b..773e46c 100644 --- a/src/include/executor/execPartition.h +++ b/src/include/executor/execPartition.h @@ -110,6 +110,7 @@ typedef struct PartitionPruneState PartitionPruningData *partprunedata[FLEXIBLE_ARRAY_MEMBER]; } PartitionPruneState; +extern size_t PartitionTupleRoutingSize(void); extern PartitionTupleRouting *ExecSetupPartitionTupleRouting(EState *estate, Relation rel); extern ResultRelInfo *ExecFindPartition(ModifyTableState *mtstate, diff --git a/src/include/pgstat.h b/src/include/pgstat.h index 7a26d6d..ccc9a93 100644 --- a/src/include/pgstat.h +++ b/src/include/pgstat.h @@ -88,6 +88,8 @@ typedef enum StatMsgType PGSTAT_MTYPE_SUBSCRIPTIONPURGE, PGSTAT_MTYPE_SUBWORKERERROR, PGSTAT_MTYPE_SUBWORKERPURGE, + PGSTAT_MTYPE_SUBWORKERXACTEND, + PGSTAT_MTYPE_SUBWORKERTWOPHASEXACT, } StatMsgType; /* ---------- @@ -591,6 +593,54 @@ typedef struct PgStat_MsgSubWorkerPurge } PgStat_MsgSubWorkerPurge; /* ---------- + * PgStat_MsgSubscriptionXactEnd Sent by the apply worker or the table sync worker + * to report successful transaction ends. + * ---------- + */ +typedef struct PgStat_MsgSubWorkerXactEnd +{ + PgStat_MsgHdr m_hdr; + + /* determine the worker entry */ + Oid m_subid; + Oid m_subrelid; + + /* + * distinguish between transaction commits and streaming transaction aborts + * that are handled without error. + */ + LogicalRepMsgType m_command; + + /* memory consumption used by transaction */ + PgStat_Counter m_xact_bytes; + +} PgStat_MsgSubWorkerXactEnd; + +/* ---------- + * PgStat_MsgSubWorkerTwophaseXact Sent by the apply worker to make size of prepared + * txn persistent over the server restart and make it + * visible after commit prepare or rollback prepared. + * This is separated from PgStat_MsgSubWorkerXactEnd + * so that we can reduce message size of gid for other + * operations (e.g. normal COMMIT) that should happen more + * frequently than prepare operation usually. + * ---------- + */ +typedef struct PgStat_MsgSubWorkerTwophaseXact +{ + PgStat_MsgHdr m_hdr; + + /* determine the subscription */ + Oid m_subid; + + LogicalRepMsgType m_command; + char m_gid[GIDSIZE]; + int gid_len; + PgStat_Counter m_xact_bytes; + +} PgStat_MsgSubWorkerTwophaseXact; + +/* ---------- * PgStat_MsgSubWorkerError Sent by the apply worker or the table sync worker to * report the error occurred during logical replication. * ---------- @@ -609,6 +659,12 @@ typedef struct PgStat_MsgSubWorkerError Oid m_subrelid; /* + * Transaction stats of subscription needs to be updated when an + * error occurs. + */ + PgStat_Counter m_xact_error_bytes; + + /* * Oids of the database and the table that the reporter was actually * processing. m_relid can be InvalidOid if an error occurred during * worker applying a non-data-modification message such as RELATION. @@ -803,6 +859,8 @@ typedef union PgStat_Msg PgStat_MsgSubscriptionPurge msg_subscriptionpurge; PgStat_MsgSubWorkerError msg_subworkererror; PgStat_MsgSubWorkerPurge msg_subworkerpurge; + PgStat_MsgSubWorkerXactEnd msg_subworkerxactend; + PgStat_MsgSubWorkerTwophaseXact msg_subworkertwophasexact; } PgStat_Msg; @@ -1035,6 +1093,16 @@ typedef struct PgStat_StatSubWorkerEntry PgStat_StatSubWorkerKey key; /* hash key (must be first) */ /* + * Transaction statistics of subscription worker + */ + PgStat_Counter xact_commit_count; + PgStat_Counter xact_commit_bytes; + PgStat_Counter xact_error_count; + PgStat_Counter xact_error_bytes; + PgStat_Counter xact_abort_count; + PgStat_Counter xact_abort_bytes; + + /* * Subscription worker error statistics representing an error that * occurred during application of logical replication or the initial table * synchronization. @@ -1049,6 +1117,22 @@ typedef struct PgStat_StatSubWorkerEntry TimestampTz stat_reset_timestamp; } PgStat_StatSubWorkerEntry; +/* prepared transaction */ +typedef struct PgStat_StatSubWorkerPreparedXact +{ + Oid subid; + char gid[GIDSIZE]; +} PgStat_StatSubWorkerPreparedXact; + +typedef struct PgStat_StatSubWorkerPreparedXactSize +{ + PgStat_StatSubWorkerPreparedXact key; /* hash key */ + + Oid subid; + char gid[GIDSIZE]; + PgStat_Counter xact_size; +} PgStat_StatSubWorkerPreparedXactSize; + /* * Working state needed to accumulate per-function-call timing statistics. */ @@ -1158,6 +1242,13 @@ extern void pgstat_report_checksum_failure(void); extern void pgstat_report_replslot(const PgStat_StatReplSlotEntry *repSlotStat); extern void pgstat_report_replslot_create(const char *slotname); extern void pgstat_report_replslot_drop(const char *slotname); +extern void pgstat_report_subworker_xact_end(Oid subid, Oid subrel, + LogicalRepMsgType command, PgStat_Counter xact_size); +extern void pgstat_report_subworker_twophase_xact(Oid subid, LogicalRepMsgType command, + PgStat_Counter xact_size, + LogicalRepPreparedTxnData *prepared_data, + LogicalRepCommitPreparedTxnData *commit_data, + LogicalRepRollbackPreparedTxnData *rollback_data); extern void pgstat_report_subworker_error(Oid subid, Oid subrelid, Oid relid, LogicalRepMsgType command, TransactionId xid, const char *errmsg); diff --git a/src/include/replication/logicalworker.h b/src/include/replication/logicalworker.h index 2ad61a0..9a8447b 100644 --- a/src/include/replication/logicalworker.h +++ b/src/include/replication/logicalworker.h @@ -16,4 +16,9 @@ extern void ApplyWorkerMain(Datum main_arg); extern bool IsLogicalWorker(void); +/* for transaction stats */ +extern int64 get_apply_error_context_xact_size(void); +extern void add_apply_error_context_xact_size(int64 size); +extern void reset_apply_error_context_xact_size(void); + #endif /* LOGICALWORKER_H */ diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out index f6b1bd6..8d7f065 100644 --- a/src/test/regress/expected/rules.out +++ b/src/test/regress/expected/rules.out @@ -2097,11 +2097,17 @@ pg_stat_subscription| SELECT su.oid AS subid, pg_stat_subscription_workers| SELECT w.subid, s.subname, w.subrelid, - w.relid, - w.command, - w.xid, - w.error_count, - w.error_message, + w.xact_commit_count, + w.xact_commit_bytes, + w.xact_error_count, + w.xact_error_bytes, + w.xact_abort_count, + w.xact_abort_bytes, + w.last_error_relid, + w.last_error_command, + w.last_error_xid, + w.last_error_count, + w.last_error_message, w.last_error_time, w.stats_reset FROM ( SELECT pg_subscription.oid AS subid, @@ -2112,7 +2118,7 @@ pg_stat_subscription_workers| SELECT w.subid, pg_subscription_rel.srrelid AS relid FROM pg_subscription_rel WHERE (pg_subscription_rel.srsubstate <> 'r'::"char")) sr, - (LATERAL pg_stat_get_subscription_worker(sr.subid, sr.relid) w(subid, subrelid, relid, command, xid, error_count, error_message, last_error_time, stats_reset) + (LATERAL pg_stat_get_subscription_worker(sr.subid, sr.relid) w(subid, subrelid, xact_commit_count, xact_commit_bytes, xact_error_count, xact_error_bytes, xact_abort_count, xact_abort_bytes, last_error_relid, last_error_command, last_error_xid, last_error_count, last_error_message, last_error_time, stats_reset) JOIN pg_subscription s ON ((w.subid = s.oid))); pg_stat_sys_indexes| SELECT pg_stat_all_indexes.relid, pg_stat_all_indexes.indexrelid, diff --git a/src/test/subscription/t/026_error_report.pl b/src/test/subscription/t/026_error_report.pl index 3d23bb5..9ff619e 100644 --- a/src/test/subscription/t/026_error_report.pl +++ b/src/test/subscription/t/026_error_report.pl @@ -15,8 +15,8 @@ sub test_subscription_error my $check_sql = qq[ SELECT count(1) > 0 FROM pg_stat_subscription_workers -WHERE relid = '$relname'::regclass]; - $check_sql .= " AND xid = '$xid'::xid;" if $xid ne ''; +WHERE last_error_relid = '$relname'::regclass]; + $check_sql .= " AND last_error_xid = '$xid'::xid;" if $xid ne ''; # Wait for the error statistics to be updated. $node->poll_query_until( @@ -26,9 +26,9 @@ WHERE relid = '$relname'::regclass]; my $result = $node->safe_psql( 'postgres', qq[ -SELECT subname, command, relid::regclass, error_count > 0 +SELECT subname, last_error_command, last_error_relid::regclass, last_error_count > 0 FROM pg_stat_subscription_workers -WHERE relid = '$relname'::regclass; +WHERE last_error_relid = '$relname'::regclass; ]); is($result, $expected_error, $msg); } diff --git a/src/test/subscription/t/027_worker_xact_stats.pl b/src/test/subscription/t/027_worker_xact_stats.pl new file mode 100644 index 0000000..0dad552 --- /dev/null +++ b/src/test/subscription/t/027_worker_xact_stats.pl @@ -0,0 +1,172 @@ + +# Copyright (c) 2021, PostgreSQL Global Development Group + +# Tests for subscription worker statistics during apply. +use strict; +use warnings; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More tests => 2; + +# Create publisher node +my $node_publisher = PostgreSQL::Test::Cluster->new('publisher'); +$node_publisher->init(allows_streaming => 'logical'); +$node_publisher->append_conf('postgresql.conf', qq[ +logical_decoding_work_mem = 64kB +max_prepared_transactions = 10 +max_wal_senders = 10 +wal_sender_timeout = 0 +]); +$node_publisher->start; + +# Create subscriber node +my $node_subscriber = PostgreSQL::Test::Cluster->new('subscriber'); +$node_subscriber->init(allows_streaming => 'logical'); +$node_subscriber->append_conf('postgresql.conf', qq[ +max_prepared_transactions = 10 +]); +$node_subscriber->start; + +# Setup structures on the publisher and the subscriber +$node_publisher->safe_psql('postgres', + "CREATE TABLE test_tab (a int)"); +$node_subscriber->safe_psql('postgres', + "CREATE TABLE test_tab (a int primary key);"); + +# Setup logical replication +my $publisher_connstr = $node_publisher->connstr . ' dbname=postgres'; +$node_publisher->safe_psql('postgres', + "CREATE PUBLICATION tap_pub FOR TABLE test_tab"); + +my $appname = 'tap_sub'; +my $subopts = 'streaming = on, two_phase = on, copy_data = false'; +$node_subscriber->safe_psql('postgres', qq[ +CREATE SUBSCRIPTION tap_sub +CONNECTION '$publisher_connstr application_name=$appname' +PUBLICATION tap_pub WITH ($subopts); +]); + +$node_publisher->wait_for_catchup($appname); + +# There's no entry at the beginning +my $result = $node_subscriber->safe_psql('postgres', +"SELECT count(*) FROM pg_stat_subscription_workers;"); +is($result, q(0), 'no entry for transaction stats yet'); + +# COMMIT +$node_publisher->safe_psql('postgres', + "BEGIN; INSERT INTO test_tab VALUES (1); COMMIT;"); + +$node_subscriber->poll_query_until('postgres', +"SELECT count(1) = 1 FROM pg_stat_subscription_workers where xact_commit_count = 1;") + or die "didn't get updates of xact stats by commit"; + +# Now, stats collector make the bytes updated also. +$result = $node_subscriber->safe_psql('postgres', +"SELECT xact_commit_bytes > 0 FROM pg_stat_subscription_workers;"); +is($result, q(t), 'got consumed bytes'); + +# STREAM COMMIT +$node_publisher->safe_psql( + 'postgres', + "BEGIN; INSERT INTO test_tab VALUES(generate_series(1001, 2000)); COMMIT;"); +$node_subscriber->poll_query_until('postgres', +"SELECT count(1) = 1 FROM pg_stat_subscription_workers where xact_commit_count = 2;") + or die "didn't get updates of xact stats by stream commit"; + +# STREAM PREPARE & COMMIT PREPARED +# This should restore the xact size before the shutdown. +$node_publisher->safe_psql( + 'postgres', q[ +BEGIN; +INSERT INTO test_tab VALUES(generate_series(2001, 3000)); +PREPARE TRANSACTION 'gid1'; +]); + +# This streamed prepare is not displayed until the commit prepared +# or rollback prepared. Hence, there's no way to confirm that +# stats collector has received the bytes of prepared transaction. +# So, instead of checking the view, issue one more committed transaction +# after the prepare and make sure that this commit's update is done, +# which should mean the previous streamed prepare is already processed +# by the stats collector as well. +$node_publisher->safe_psql('postgres', +"BEGIN; INSERT INTO test_tab VALUES (2); COMMIT;"); +$node_subscriber->poll_query_until('postgres', +"SELECT count(1) = 1 FROM pg_stat_subscription_workers where xact_commit_count = 3;") + or die "didn't process the updates of committed transaction"; + +$node_subscriber->restart; + +# Get xact_commit_bytes before commit prepared. +my $tmp = $node_subscriber->safe_psql('postgres', +"SELECT xact_commit_bytes FROM pg_stat_subscription_workers where xact_commit_count = 3;"); + +# Commit prepared increments the xact_commit. +$node_publisher->safe_psql('postgres', "COMMIT PREPARED 'gid1'"); +$node_subscriber->poll_query_until('postgres', +"SELECT count(1) = 1 FROM pg_stat_subscription_workers where xact_commit_count = 4;") + or die "didn't get updates of xact stats by stream prepare and commit prepared"; + +$node_subscriber->poll_query_until('postgres', +"SELECT xact_commit_bytes > $tmp FROM pg_stat_subscription_workers where xact_commit_count = 4;"); + +# STREAM ABORT +# Store previous stream_counts to recognize +# another streaming from the increase of this value below. +$tmp = $node_publisher->safe_psql('postgres', +"SELECT stream_count FROM pg_stat_replication_slots"); + +# Cause a new streaming and check it by another session +my $in = ''; +my $out = ''; +my $timer = IPC::Run::timeout(180); +my $h = $node_publisher->background_psql('postgres', \$in, \$out, $timer, + on_error_stop => 1); + +$in .= q{ +BEGIN; +INSERT INTO test_tab VALUES(generate_series(3001, 4000)); +}; +$h->pump_nb; + +# Wait until this transaction is streamed certainly +# and after that rollback to send stream abort. +$node_publisher->poll_query_until('postgres', +"SELECT stream_count > $tmp FROM pg_stat_replication_slots;") + or die "didn't stream data of a new transaction"; + +$in .= q{ +ROLLBACK; +\q +}; +$h->finish; + +$node_subscriber->poll_query_until('postgres', +"SELECT count(1) = 1 FROM pg_stat_subscription_workers where xact_abort_count = 1;") + or die "didn't get updates of xact stats by stream abort"; + +# ROLLBACK PREPARED +$node_publisher->safe_psql('postgres', q[ +BEGIN; +INSERT INTO test_tab VALUES (3); +PREPARE TRANSACTION 'gid2'; +]); +$node_publisher->safe_psql('postgres', "ROLLBACK PREPARED 'gid2'"); +$node_subscriber->poll_query_until('postgres', +"SELECT count(1) = 1 FROM pg_stat_subscription_workers where xact_abort_count = 2;") + or die "didn't get updates of xact stats by rollback prepared"; + +# error stats (by duplication error) +$node_publisher->safe_psql('postgres', q[ +BEGIN; +INSERT INTO test_tab VALUES (1); +COMMIT; +]); + +$node_subscriber->poll_query_until('postgres', +"SELECT count(1) = 1 FROM pg_stat_subscription_workers where xact_error_count > 0;") + or die "didn't get updates of xact stats by error"; + +$node_subscriber->stop('fast'); +$node_publisher->stop('fast'); -- 2.2.0 ^ permalink raw reply [nested|flat] 113+ messages in thread
* Re: Failed transaction statistics to measure the logical replication progress @ 2021-11-08 06:12 vignesh C <[email protected]> parent: [email protected] <[email protected]> 2 siblings, 1 reply; 113+ messages in thread From: vignesh C @ 2021-11-08 06:12 UTC (permalink / raw) To: [email protected] <[email protected]>; +Cc: Greg Nancarrow <[email protected]>; Masahiko Sawada <[email protected]>; Amit Kapila <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]> On Fri, Nov 5, 2021 at 1:42 PM [email protected] <[email protected]> wrote: > > On Thursday, November 4, 2021 9:54 AM Greg Nancarrow <[email protected]> wrote: > > On Tue, Nov 2, 2021 at 12:18 AM [email protected] > > <[email protected]> wrote: > > > > > > On Thursday, October 28, 2021 11:19 PM I wrote: > > > > I've created a new patch that extends pg_stat_subscription_workers > > > > to include other transaction statistics. > > > > > > > > Note that this patch depends on v18 patch-set in [1]... > > > Rebased based on the v19 in [1]. > > > Also, fixed documentation a little and made tests tidy. > > > FYI, the newly included TAP test(027_worker_xact_stats.pl) is stable > > > because I checked that 100 times of its execution in a tight loop all passed. > > > > > > > I have done some basic testing of the patch and have some initial review > > comments: > Thanks for your review ! > > > (1) I think this patch needs to be in "git format-patch" format, with a proper > > commit message that describes the purpose of the patch and the functionality > > it adds, and any high-level design points (something like the overview given in > > the initial post, and accounting for the subsequent discussion points and > > updated functionality). > Fixed. > > > (2) doc/src/sgml/monitoring.sgml > > There are some grammatical issues in the current description. I suggest > > changing it to something like: > > BEFORE: > > + <entry>At least one row per subscription, showing about > > transaction statistics and error summary that > > AFTER: > > + <entry>At least one row per subscription, showing transaction > > statistics and information about errors that > Fixed. > > > (2) doc/src/sgml/monitoring.sgml > > The current description seems a little confusing. > > Per subscription, it shows the transaction statistics and any last error info from > > tablesync/apply workers? If this is the case, I'd suggest the following change: > > > > BEFORE: > > + one row per subscription for transaction statistics and summary of the > > last > > + error reported by workers applying logical replication changes and > > workers > > + handling the initial data copy of the subscribed tables. > > AFTER: > > + one row per subscription, showing corresponding transaction statistics > > and > > + information about the last error reported by workers applying > > logical replication > > + changes or by workers handling the initial data copy of the > > subscribed tables. > Fixed. > > > (3) xact_commit > > I think that the "xact_commit" column should be named "xact_commit_count" > > or "xact_commits". > > Similarly, I think "xact_error" should be named "xact_error_count" or > > "xact_errors", and "xact_aborts" should be named "xact_abort_count" or > > "xact_aborts". > I prefered *_count. Renamed. > > > (4) xact_commit_bytes > > > > + Amount of transactions data successfully applied in this subscription. > > + Consumed memory for xact_commit is displayed. > > > > I find this description a bit confusing. "Consumed memory for xact_commit" > > seems different to "transactions data". > > Could the description be something like: Amount of data (in bytes) successfully > > applied in this subscription, across "xact_commit_count" > > transactions. > Fixed. > > > (5) > > I'd suggest some minor rewording for the following: > > > > BEFORE: > > + Number of transactions failed to be applied and caught by table > > + sync worker or main apply worker in this subscription. > > AFTER: > > + Number of transactions that failed to be applied by the table > > + sync worker or main apply worker in this subscription. > Fixed. > > > (6) xact_error_bytes > > Again, it's a little confusing referring to "consumed memory" here. > > How about rewording this, something like: > > > > BEFORE: > > + Amount of transactions data unsuccessfully applied in this > > subscription. > > + Consumed memory that past failed transaction used is displayed. > > AFTER: > > + Amount of data (in bytes) unsuccessfully applied in this > > subscription by the last failed transaction. > xact_error_bytes (and other bytes columns as well) is cumulative > so when a new error happens, the size of this new bytes would be > added to the same. So here we shouldn't mention just the last error. > I simply applied your previous comments of 'xact_commit_bytes' > to 'xact_error_bytes' description. > > > (7) > > The additional information provided for "xact_abort_bytes" needs some > > rewording, something like: > > > > BEFORE: > > + Increase <literal>logical_decoding_work_mem</literal> on the > > publisher > > + so that it exceeds the size of whole streamed transaction > > + to suppress unnecessary consumed network bandwidth in addition to > > change > > + in memory of the subscriber, if unexpected amount of streamed > > transactions > > + are aborted. > > AFTER: > > + In order to suppress unnecessary consumed network bandwidth, > > increase > > + <literal>logical_decoding_work_mem</literal> on the publisher so > > that it > > + exceeds the size of the whole streamed transaction, and > > additionally increase > > + the available subscriber memory, if an unexpected amount of > > streamed transactions > > + are aborted. > > I'm not sure about the last part. > > additionally increase the available subscriber memory, > Which GUC parameter did you mean by this ? > Could we point out and enalrge the memory size only for > subscriber's apply processing intentionally ? > I incorporated (7) except for this last part. > Will revise according to your reply. > > I also added the explanation about > xact_abort_bytes itself to align with other bytes columns. > > > (8) > > Suggested update: > > > > BEFORE: > > + * Tell the collector that worker transaction has finished without problem. > > AFTER: > > + * Tell the collector that the worker transaction has successfully completed. > Fixed. > > > (9) src/backend/postmaster/pgstat.c > > I think that the GID copying is unnecessarily copying the whole GID buffer or > > using an additional strlen(). > > It should be changed to use strlcpy() to match other code: > > > > BEFORE: > > + /* get the gid for this two phase operation */ if (command == > > + LOGICAL_REP_MSG_PREPARE || > > + command == LOGICAL_REP_MSG_STREAM_PREPARE) > > + memcpy(msg.m_gid, prepare_data->gid, GIDSIZE); else if (command == > > + LOGICAL_REP_MSG_COMMIT_PREPARED) > > + memcpy(msg.m_gid, commit_data->gid, GIDSIZE); else /* rollback > > + prepared */ > > + memcpy(msg.m_gid, rollback_data->gid, GIDSIZE); > > AFTER: > > + /* get the gid for this two phase operation */ if (command == > > + LOGICAL_REP_MSG_PREPARE || > > + command == LOGICAL_REP_MSG_STREAM_PREPARE) > > + strlcpy(msg.m_gid, prepare_data->gid, sizeof(msg.m_gid)); else if > > + (command == LOGICAL_REP_MSG_COMMIT_PREPARED) > > + strlcpy(msg.m_gid, commit_data->gid, sizeof(msg.m_gid)); else /* > > + rollback prepared */ > > + strlcpy(msg.m_gid, rollback_data->gid, sizeof(msg.m_gid)); > Fixed. > > > > > BEFORE: > > + strlcpy(prepared_txn->gid, msg->m_gid, strlen(msg->m_gid) + 1); > > AFTER: > > + strlcpy(prepared_txn->gid, msg->m_gid, sizeof(prepared_txn->gid)); > > > > BEFORE: > > + memcpy(key.gid, msg->m_gid, strlen(msg->m_gid)); > > AFTER: > > + strlcpy(key.gid, msg->m_gid, sizeof(key.gid)); > > > > BEFORE: > > + memcpy(key.gid, gid, strlen(gid)); > > AFTER: > > + strlcpy(key.gid, gid, sizeof(key.gid)); > Fixed. > > > (10) src/backend/replication/logical/worker.c > > Some suggested rewording: > > > > BEFORE: > > + * size of streaming transaction resources because it have used the > > AFTER: > > + * size of streaming transaction resources because it has used the > Fixed. > > > BEFORE: > > + * tradeoff should not be good. Also, add multiple values > > + * at once in order to reduce the number of this function call. > > AFTER: > > + * tradeoff would not be good. Also, add multiple values > > + * at once in order to reduce the number of calls to this function. > Fixed. > > > (11) update_apply_change_size() > > Shouldn't this function be declared static? > Fixed. > > > (12) stream_write_change() > > > > + streamed_entry->xact_size = streamed_entry->xact_size + total_len; > > /* update */ > > > > could be simply written as: > > > > + streamed_entry->xact_size += total_len; /* update */ > Fixed. > > Lastly, I removed one unnecessary test that > checked publisher's stats in the TAP tests. > Also I introduced ApplyTxnExtraData structure to > remove void* argument of update_apply_change_size > that might worsen the readability of codes > in the previous version. Thanks for the updated patch. Few comments: 1) You could remove LogicalRepPreparedTxnData, LogicalRepCommitPreparedTxnData & LogicalRepRollbackPreparedTxnData and change it to char *gid to reduce the function parameter and simiplify the assignment: + */ +void +pgstat_report_subworker_twophase_xact(Oid subid, LogicalRepMsgType command, + PgStat_Counter xact_size, + LogicalRepPreparedTxnData *prepare_data, + LogicalRepCommitPreparedTxnData *commit_data, + LogicalRepRollbackPreparedTxnData *rollback_data) 2) Shouldn't this change be part of skip xid patch? - TupleDescInitEntry(tupdesc, (AttrNumber) 4, "command", + TupleDescInitEntry(tupdesc, (AttrNumber) 10, "last_error_command", TEXTOID, -1, 0); - TupleDescInitEntry(tupdesc, (AttrNumber) 5, "xid", + TupleDescInitEntry(tupdesc, (AttrNumber) 11, "last_error_xid", XIDOID, -1, 0); - TupleDescInitEntry(tupdesc, (AttrNumber) 6, "error_count", + TupleDescInitEntry(tupdesc, (AttrNumber) 12, "last_error_count", INT8OID, -1, 0); - TupleDescInitEntry(tupdesc, (AttrNumber) 7, "error_message", + TupleDescInitEntry(tupdesc, (AttrNumber) 13, "last_error_message", TEXTOID, -1, 0); - TupleDescInitEntry(tupdesc, (AttrNumber) 8, "last_error_time", + TupleDescInitEntry(tupdesc, (AttrNumber) 14, "last_error_time", 3) This newly added structures should be added to typedefs.list: ApplyTxnExtraData XactSizeEntry PgStat_MsgSubWorkerXactEnd PgStat_MsgSubWorkerTwophaseXact PgStat_StatSubWorkerPreparedXact PgStat_StatSubWorkerPreparedXactSize 4) We are not sending the transaction size in case of table sync, is this intentional, if so I felt we should document this in pg_stat_subscription_workers + /* Report the success of table sync. */ + pgstat_report_subworker_xact_end(MyLogicalRepWorker->subid, + MyLogicalRepWorker->relid, + 0 /* no logical message type */, + 0 /* xact size */); + 5) pg_stat_subscription_workers has a lot of columns, if we can reduce the column size the readability will improve, like xact_commit_count to commit_count, xact_commit_bytes to commit_bytes, etc + w.xact_commit_count, + w.xact_commit_bytes, + w.xact_error_count, + w.xact_error_bytes, + w.xact_abort_count, + w.xact_abort_bytes, Regards, Vignesh ^ permalink raw reply [nested|flat] 113+ messages in thread
* Re: Failed transaction statistics to measure the logical replication progress @ 2021-11-08 23:50 Greg Nancarrow <[email protected]> parent: [email protected] <[email protected]> 2 siblings, 0 replies; 113+ messages in thread From: Greg Nancarrow @ 2021-11-08 23:50 UTC (permalink / raw) To: [email protected] <[email protected]>; +Cc: Masahiko Sawada <[email protected]>; Amit Kapila <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]> On Fri, Nov 5, 2021 at 7:11 PM [email protected] <[email protected]> wrote: > > I'm not sure about the last part. > > additionally increase the available subscriber memory, > Which GUC parameter did you mean by this ? > Could we point out and enalrge the memory size only for > subscriber's apply processing intentionally ? > I incorporated (7) except for this last part. > Will revise according to your reply. > I might have misinterpreted your original description, so I'll re-review that in your latest patch. As a newer version (v20) of the prerequisite patch was posted a day ago, it looks like your patch needs to be rebased against that (as it currently applies on top of the v19 version only). Regards, Greg Nancarrow Fujitsu Australia ^ permalink raw reply [nested|flat] 113+ messages in thread
* Re: Failed transaction statistics to measure the logical replication progress @ 2021-11-09 03:07 Greg Nancarrow <[email protected]> parent: [email protected] <[email protected]> 2 siblings, 1 reply; 113+ messages in thread From: Greg Nancarrow @ 2021-11-09 03:07 UTC (permalink / raw) To: [email protected] <[email protected]>; +Cc: Masahiko Sawada <[email protected]>; Amit Kapila <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]> On Fri, Nov 5, 2021 at 7:11 PM [email protected] <[email protected]> wrote: > I did a quick scan through the latest v8 patch and noticed the following things: src/backend/postmaster/pgstat.c (1) pgstat_recv_subworker_twophase_xact() The copying from msg->m_gid to key.gid does not seem to be correct. strlen() is being called on a junk value, since key.gid has not been assigned yet. It should be changed as follows: BEFORE: + strlcpy(key.gid, msg->m_gid, strlen(key.gid)); AFTER: + strlcpy(key.gid, msg->m_gid, sizeof(key.gid)); (2) pgstat_get_subworker_prepared_txn() Similar to above, strlen() usage is not correct, and should use strlcpy() instead of memcpy(). BEFORE: + memcpy(key.gid, gid, strlen(key.gid)); AFTER: + strlcpy(key.gid, gid, sizeof(key.gid)); (3) stats_reset Note that the "stats_reset" column has been removed from the pg_stat_subscription_workers view in the underlying latest v20 patch. Regards, Greg Nancarrow Fujitsu Australia ^ permalink raw reply [nested|flat] 113+ messages in thread
* RE: Failed transaction statistics to measure the logical replication progress @ 2021-11-09 11:35 [email protected] <[email protected]> parent: Greg Nancarrow <[email protected]> 0 siblings, 3 replies; 113+ messages in thread From: [email protected] @ 2021-11-09 11:35 UTC (permalink / raw) To: 'Greg Nancarrow' <[email protected]>; +Cc: Masahiko Sawada <[email protected]>; Amit Kapila <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]> On Tuesday, November 9, 2021 12:08 PM Greg Nancarrow <[email protected]> wrote: > On Fri, Nov 5, 2021 at 7:11 PM [email protected] > <[email protected]> wrote: > > > > I did a quick scan through the latest v8 patch and noticed the following things: I appreciate your review ! > src/backend/postmaster/pgstat.c > > (1) pgstat_recv_subworker_twophase_xact() > The copying from msg->m_gid to key.gid does not seem to be correct. > strlen() is being called on a junk value, since key.gid has not been assigned yet. > It should be changed as follows: > > BEFORE: > + strlcpy(key.gid, msg->m_gid, strlen(key.gid)); > AFTER: > + strlcpy(key.gid, msg->m_gid, sizeof(key.gid)); Fixed. > (2) pgstat_get_subworker_prepared_txn() > Similar to above, strlen() usage is not correct, and should use > strlcpy() instead of memcpy(). > > BEFORE: > + memcpy(key.gid, gid, strlen(key.gid)); > AFTER: > + strlcpy(key.gid, gid, sizeof(key.gid)); Fixed. > (3) stats_reset > Note that the "stats_reset" column has been removed from the > pg_stat_subscription_workers view in the underlying latest v20 patch. Yes. I've rebased and updated the patch, paying attention to this point. Attached the updated version. Best Regards, Takamichi Osumi Attachments: [application/octet-stream] extend_xact_stats_of_subscription_worker_v9.patch (60.0K, ../../TYCPR01MB8373B1614522B90BA96883E0ED929@TYCPR01MB8373.jpnprd01.prod.outlook.com/2-extend_xact_stats_of_subscription_worker_v9.patch) download | inline diff: From 4a8e0913d003928bd4726ed4da2ab15bf09cecb3 Mon Sep 17 00:00:00 2001 From: Osumi Takamichi <[email protected]> Date: Tue, 9 Nov 2021 11:08:01 +0000 Subject: [PATCH v9] Extend pg_stat_subscription_workers to include general transaction statistics Categorize transactions of logical replication subscriber into three types (commit, error, abort) and introduce cumulative columns of those numbers and amounts of consumed data during message apply respectively. One scenario to utilize those columns is to suppress unnecessary network bandwidth, when streaming transaction aborted more than expected is observed by the column of abort_count or its bytes column. The calculation of consumed resources by subscriber is computed based on the data structure for message apply, which is different from that of publisher's decoding processing. Stats of prepared transaction becomes persistent to conclude the appropriate category for the transaction at either commit prepared or rollback prepared time. Also, streaming transactions running in parallel on publisher can cause data blocks in unpredictable order. Managing each streaming tranaction by xid makes it possible to handle stream abort, stream commit and stream prepare properly. Author: Takamichi Osumi Discussed & Reviewed-by: Amit Kapila, Masahiko Sawada, Hou Zhijie, Greg Nancarrow, Vignesh C, Ajin Cherian Discussion: https://www.postgresql.org/message-id/OSBPR01MB48887CA8F40C8D984A6DC00CED199%40OSBPR01MB4888.jpnprd01.prod.outlook.com --- doc/src/sgml/monitoring.sgml | 95 +++++-- src/backend/catalog/system_views.sql | 18 +- src/backend/executor/execPartition.c | 10 + src/backend/postmaster/pgstat.c | 294 ++++++++++++++++++++++ src/backend/replication/logical/tablesync.c | 6 + src/backend/replication/logical/worker.c | 304 +++++++++++++++++++++++ src/backend/utils/adt/pgstatfuncs.c | 44 +++- src/include/catalog/pg_proc.dat | 6 +- src/include/executor/execPartition.h | 1 + src/include/pgstat.h | 90 +++++++ src/include/replication/logicalworker.h | 5 + src/test/regress/expected/rules.out | 16 +- src/test/subscription/t/026_error_report.pl | 8 +- src/test/subscription/t/027_worker_xact_stats.pl | 173 +++++++++++++ src/tools/pgindent/typedefs.list | 6 + 15 files changed, 1032 insertions(+), 44 deletions(-) create mode 100644 src/test/subscription/t/027_worker_xact_stats.pl diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml index 7a4a1a2..3a35efa 100644 --- a/doc/src/sgml/monitoring.sgml +++ b/doc/src/sgml/monitoring.sgml @@ -629,7 +629,7 @@ postgres 27093 0.0 0.0 30096 2752 ? Ss 11:34 0:00 postgres: ser <row> <entry><structname>pg_stat_subscription_workers</structname><indexterm><primary>pg_stat_subscription_workers</primary></indexterm></entry> - <entry>At least one row per subscription, showing about errors that + <entry>At least one row per subscription, showing transaction statistics and information about errors that occurred on subscription. See <link linkend="monitoring-pg-stat-subscription-workers"> <structname>pg_stat_subscription_workers</structname></link> for details. @@ -3052,9 +3052,10 @@ SELECT pid, wait_event_type, wait_event FROM pg_stat_activity WHERE wait_event i <para> The <structname>pg_stat_subscription_workers</structname> view will contain - one row per subscription error reported by workers applying logical - replication changes and workers handling the initial data copy of the - subscribed tables. + one row per subscription, showing corresponding transaction statistics and + information about the last error reported by workers applying logical replication + changes or by workers handling the initial data copy of the subscribed tables. + The statistics of transaction size is utilized only by the apply worker. </para> <table id="pg-stat-subscription-workers" xreflabel="pg_stat_subscription_workers"> @@ -3102,20 +3103,86 @@ SELECT pid, wait_event_type, wait_event FROM pg_stat_activity WHERE wait_event i <row> <entry role="catalog_table_entry"><para role="column_definition"> - <structfield>relid</structfield> <type>oid</type> + <structfield>commit_count</structfield> <type>bigint</type> + </para> + <para> + Number of transactions successfully applied in this subscription. + COMMIT, COMMIT of streaming transaction and COMMIT PREPARED increments + this counter. + </para></entry> + </row> + + <row> + <entry role="catalog_table_entry"><para role="column_definition"> + <structfield>commit_bytes</structfield> <type>bigint</type> + </para> + <para> + Amount of data (in bytes) successfully applied in this subscription, + across <literal>commit_count</literal> transactions. + </para></entry> + </row> + + <row> + <entry role="catalog_table_entry"><para role="column_definition"> + <structfield>error_count</structfield> <type>bigint</type> + </para> + <para> + Number of transactions that failed to be applied by the table + sync worker or main apply worker in this subscription. + </para></entry> + </row> + + <row> + <entry role="catalog_table_entry"><para role="column_definition"> + <structfield>error_bytes</structfield> <type>bigint</type> + </para> + <para> + Amount of data (in bytes) unsuccessfully applied in this subscription, + across <literal>error_count</literal> transactions. + </para></entry> + </row> + + <row> + <entry role="catalog_table_entry"><para role="column_definition"> + <structfield>abort_count</structfield> <type>bigint</type> + </para> + <para> + Number of transactions aborted in this subscription. + ROLLBACK PREPARED and abort of streaming transaction + increments this counter. + </para></entry> + </row> + + <row> + <entry role="catalog_table_entry"><para role="column_definition"> + <structfield>abort_bytes</structfield> <type>bigint</type> + </para> + <para> + Amount of data (in bytes) aborted in this subscription, + across <literal>abort_count</literal> transactions. + In order to suppress unnecessary consumed network bandwidth, + increase <literal>logical_decoding_work_mem</literal> on the publisher + so that it exceeds the size of the whole streamed transaction, if + an unexpected amount of streamed transactions are aborted. + </para></entry> + </row> + + <row> + <entry role="catalog_table_entry"><para role="column_definition"> + <structfield>last_error_relid</structfield> <type>oid</type> </para> <para> OID of the relation that the worker was processing when the - error occurred + last error occurred </para></entry> </row> <row> <entry role="catalog_table_entry"><para role="column_definition"> - <structfield>command</structfield> <type>text</type> + <structfield>last_error_command</structfield> <type>text</type> </para> <para> - Name of command being applied when the error occurred. This field + Name of last command being applied when the error occurred. This field is always NULL if the error was reported during the initial data copy. </para></entry> @@ -3123,10 +3190,10 @@ SELECT pid, wait_event_type, wait_event FROM pg_stat_activity WHERE wait_event i <row> <entry role="catalog_table_entry"><para role="column_definition"> - <structfield>xid</structfield> <type>xid</type> + <structfield>last_error_xid</structfield> <type>xid</type> </para> <para> - Transaction ID of the publisher node being applied when the error + Transaction ID of the publisher node being applied when the last error occurred. This field is always NULL if the error was reported during the initial data copy. </para></entry> @@ -3134,19 +3201,19 @@ SELECT pid, wait_event_type, wait_event FROM pg_stat_activity WHERE wait_event i <row> <entry role="catalog_table_entry"><para role="column_definition"> - <structfield>error_count</structfield> <type>uint8</type> + <structfield>last_error_count</structfield> <type>uint8</type> </para> <para> - Number of consecutive times the error occurred + Number of consecutive times the last error occurred </para></entry> </row> <row> <entry role="catalog_table_entry"><para role="column_definition"> - <structfield>error_message</structfield> <type>text</type> + <structfield>last_error_message</structfield> <type>text</type> </para> <para> - The error message + The last error message </para></entry> </row> diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index fece48a..9e0cfce 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1267,12 +1267,18 @@ CREATE VIEW pg_stat_subscription_workers AS w.subid, s.subname, w.subrelid, - w.relid, - w.command, - w.xid, - w.error_count, - w.error_message, - w.last_error_time + w.commit_count, + w.commit_bytes, + w.error_count, + w.error_bytes, + w.abort_count, + w.abort_bytes, + w.last_error_relid, + w.last_error_command, + w.last_error_xid, + w.last_error_count, + w.last_error_message, + w.last_error_time FROM (SELECT oid as subid, NULL as relid diff --git a/src/backend/executor/execPartition.c b/src/backend/executor/execPartition.c index 5c723bc..20184c6 100644 --- a/src/backend/executor/execPartition.c +++ b/src/backend/executor/execPartition.c @@ -192,6 +192,16 @@ static void find_matching_subplans_recurse(PartitionPruningData *prunedata, bool initial_prune, Bitmapset **validsubplans); +/* + * PartitionTupleRoutingSize - exported to calculate total data size + * of logical replication mesage apply, because this is one of the + * ApplyExecutionData struct members. + */ +size_t +PartitionTupleRoutingSize(void) +{ + return sizeof(PartitionTupleRouting); +} /* * ExecSetupPartitionTupleRouting - sets up information needed during diff --git a/src/backend/postmaster/pgstat.c b/src/backend/postmaster/pgstat.c index d944361..09c1662 100644 --- a/src/backend/postmaster/pgstat.c +++ b/src/backend/postmaster/pgstat.c @@ -54,6 +54,7 @@ #include "postmaster/fork_process.h" #include "postmaster/interrupt.h" #include "postmaster/postmaster.h" +#include "replication/logicalworker.h" #include "replication/slot.h" #include "replication/walsender.h" #include "storage/backendid.h" @@ -287,6 +288,15 @@ static PgStat_SLRUStats slruStats[SLRU_NUM_ELEMENTS]; static HTAB *replSlotStatHash = NULL; /* + * Stats of prepared transactions should be displayed + * at either commit prepared or rollback prepared time, even when it's + * after the server restart. We have the apply worker send those statistics + * to the stats collector at prepare time and the startup process restore + * those at restart if necessary. + */ +static HTAB *subWorkerPreparedXactSizeHash = NULL; + +/* * List of OIDs of databases we need to write out. If an entry is InvalidOid, * it means to write only the shared-catalog stats ("DB 0"); otherwise, we * will write both that DB's data and the shared stats. @@ -326,6 +336,9 @@ static PgStat_StatTabEntry *pgstat_get_tab_entry(PgStat_StatDBEntry *dbentry, static PgStat_StatSubWorkerEntry *pgstat_get_subworker_entry(PgStat_StatDBEntry *dbentry, Oid subid, Oid subrelid, bool create); +static PgStat_StatSubWorkerPreparedXactSize *pgstat_get_subworker_prepared_txn(Oid subid, + char *gid, bool create); + static void pgstat_write_statsfiles(bool permanent, bool allDbs); static void pgstat_write_db_statsfile(PgStat_StatDBEntry *dbentry, bool permanent); static HTAB *pgstat_read_statsfiles(Oid onlydb, bool permanent, bool deep); @@ -385,6 +398,9 @@ static void pgstat_recv_replslot(PgStat_MsgReplSlot *msg, int len); static void pgstat_recv_tempfile(PgStat_MsgTempFile *msg, int len); static void pgstat_recv_subscription_purge(PgStat_MsgSubscriptionPurge *msg, int len); static void pgstat_recv_subworker_error(PgStat_MsgSubWorkerError *msg, int len); +static void pgstat_recv_subworker_xact_end(PgStat_MsgSubWorkerXactEnd *msg, int len); +static void pgstat_recv_subworker_twophase_xact(PgStat_MsgSubWorkerTwophaseXact *msg, int len); + /* ------------------------------------------------------------ * Public functions called from postmaster follow @@ -1965,6 +1981,62 @@ pgstat_report_replslot_drop(const char *slotname) } /* ---------- + * pgstat_report_subworker_xact_end() - + * + * Tell the collector that worker transaction has successfully completed. + * ---------- + */ +void +pgstat_report_subworker_xact_end(Oid subid, Oid subrel, + LogicalRepMsgType command, PgStat_Counter xact_size) +{ + PgStat_MsgSubWorkerXactEnd msg; + + Assert(command == 0 /* table sync worker */ || + command == LOGICAL_REP_MSG_COMMIT || + command == LOGICAL_REP_MSG_STREAM_ABORT || + command == LOGICAL_REP_MSG_STREAM_COMMIT); + + pgstat_setheader(&msg.m_hdr, PGSTAT_MTYPE_SUBWORKERXACTEND); + msg.m_databaseid = MyDatabaseId; + msg.m_subid = subid; + msg.m_subrelid = subrel; + msg.m_command = command; + msg.m_xact_bytes = xact_size; + pgstat_send(&msg, sizeof(PgStat_MsgSubWorkerXactEnd)); + + reset_apply_error_context_xact_size(); +} + +/* ---------- + * pgstat_report_subworker_twophase_xact() - + * + * Tell the collector that worker transaction has done 2PC related operation. + * ---------- + */ +void +pgstat_report_subworker_twophase_xact(Oid subid, LogicalRepMsgType command, + PgStat_Counter xact_size, char *gid) +{ + PgStat_MsgSubWorkerTwophaseXact msg; + + Assert(command == LOGICAL_REP_MSG_PREPARE || + command == LOGICAL_REP_MSG_STREAM_PREPARE || + command == LOGICAL_REP_MSG_COMMIT_PREPARED || + command == LOGICAL_REP_MSG_ROLLBACK_PREPARED); + + /* setup the message */ + pgstat_setheader(&msg.m_hdr, PGSTAT_MTYPE_SUBWORKERTWOPHASEXACT); + msg.m_databaseid = MyDatabaseId; + msg.m_subid = subid; + msg.m_command = command; + strlcpy(msg.m_gid, gid, sizeof(msg.m_gid)); + msg.m_xact_bytes = xact_size; + pgstat_send(&msg, sizeof(PgStat_MsgSubWorkerTwophaseXact)); + reset_apply_error_context_xact_size(); +} + +/* ---------- * pgstat_report_subworker_error() - * * Tell the collector about the subscription worker error. @@ -1985,6 +2057,7 @@ pgstat_report_subworker_error(Oid subid, Oid subrelid, Oid relid, msg.m_databaseid = MyDatabaseId; msg.m_subid = subid; msg.m_subrelid = subrelid; + msg.m_xact_error_bytes = get_apply_error_context_xact_size(); msg.m_relid = relid; msg.m_command = command; msg.m_xid = xid; @@ -1992,6 +2065,8 @@ pgstat_report_subworker_error(Oid subid, Oid subrelid, Oid relid, strlcpy(msg.m_message, errmsg, PGSTAT_SUBWORKERERROR_MSGLEN); pgstat_send(&msg, len); + + reset_apply_error_context_xact_size(); } /* ---------- @@ -3750,6 +3825,14 @@ PgstatCollectorMain(int argc, char *argv[]) pgstat_recv_subworker_error(&msg.msg_subworkererror, len); break; + case PGSTAT_MTYPE_SUBWORKERXACTEND: + pgstat_recv_subworker_xact_end(&msg.msg_subworkerxactend, len); + break; + + case PGSTAT_MTYPE_SUBWORKERTWOPHASEXACT: + pgstat_recv_subworker_twophase_xact(&msg.msg_subworkertwophasexact, len); + break; + default: break; } @@ -4058,6 +4141,22 @@ pgstat_write_statsfiles(bool permanent, bool allDbs) } /* + * Write subscription worker's prepared transaction struct + */ + if (subWorkerPreparedXactSizeHash) + { + PgStat_StatSubWorkerPreparedXactSize *prepared_size; + + hash_seq_init(&hstat, subWorkerPreparedXactSizeHash); + while((prepared_size = (PgStat_StatSubWorkerPreparedXactSize *) hash_seq_search(&hstat)) != NULL) + { + fputc('P', fpout); + rc = fwrite(prepared_size, sizeof(PgStat_StatSubWorkerPreparedXactSize), 1, fpout); + (void) rc; /* we'll check for error with ferror */ + } + } + + /* * No more output to be done. Close the temp file and replace the old * pgstat.stat with it. The ferror() check replaces testing for error * after each individual fputc or fwrite above. @@ -4541,6 +4640,40 @@ pgstat_read_statsfiles(Oid onlydb, bool permanent, bool deep) break; } + case 'P': + { + PgStat_StatSubWorkerPreparedXactSize buff; + PgStat_StatSubWorkerPreparedXactSize *prepared_xact_size; + + if (fread(&buff, 1, sizeof(PgStat_StatSubWorkerPreparedXactSize), + fpin) != sizeof(PgStat_StatSubWorkerPreparedXactSize)) + { + ereport(pgStatRunningInCollector ? LOG : WARNING, + (errmsg("corrupted statistics file \"%s\"", + statfile))); + goto done; + } + + if (subWorkerPreparedXactSizeHash == NULL) + { + HASHCTL hash_ctl; + + hash_ctl.keysize = sizeof(PgStat_StatSubWorkerPreparedXact); + hash_ctl.entrysize = sizeof(PgStat_StatSubWorkerPreparedXactSize); + hash_ctl.hcxt = pgStatLocalContext; + subWorkerPreparedXactSizeHash = hash_create("Subscription worker stats of prepared txn", + PGSTAT_SUBWORKER_HASH_SIZE, + &hash_ctl, + HASH_ELEM | HASH_STRINGS | HASH_CONTEXT); + } + + prepared_xact_size = + (PgStat_StatSubWorkerPreparedXactSize *) hash_search(subWorkerPreparedXactSizeHash, + (void *) &buff.key, + HASH_ENTER, NULL); + memcpy(prepared_xact_size, &buff, sizeof(PgStat_StatSubWorkerPreparedXactSize)); + break; + } case 'E': goto done; @@ -6148,6 +6281,112 @@ pgstat_recv_subscription_purge(PgStat_MsgSubscriptionPurge *msg, int len) } /* ---------- + * pgstat_recv_subworker_xact_end() - + * + * Process a SUBWORKERXACTEND message. + * ---------- + */ +static void +pgstat_recv_subworker_xact_end(PgStat_MsgSubWorkerXactEnd *msg, int len) +{ + PgStat_StatDBEntry *dbentry; + PgStat_StatSubWorkerEntry *wentry; + + dbentry = pgstat_get_db_entry(msg->m_databaseid, true); + wentry = pgstat_get_subworker_entry(dbentry, msg->m_subid, + msg->m_subrelid, true); + Assert(wentry); + + /* table sync worker */ + if (msg->m_command == 0) + wentry->xact_commit_count++; + else + { + /* apply worker */ + switch(msg->m_command) + { + case LOGICAL_REP_MSG_COMMIT: + case LOGICAL_REP_MSG_STREAM_COMMIT: + wentry->xact_commit_count++; + wentry->xact_commit_bytes += msg->m_xact_bytes; + break; + case LOGICAL_REP_MSG_STREAM_ABORT: + wentry->xact_abort_count++; + wentry->xact_abort_bytes += msg->m_xact_bytes; + break; + default: + elog(ERROR, "unexpected logical message type as normal apply end"); + break; + } + } +} + +/* ---------- + * pgstat_recv_subworker_twophase_xact() - + * + * Process a SUBWORKERTWOPHASEXACT message. + * ---------- + */ +static void +pgstat_recv_subworker_twophase_xact(PgStat_MsgSubWorkerTwophaseXact *msg, int len) +{ + PgStat_StatSubWorkerPreparedXactSize *prepared_txn; + PgStat_StatDBEntry *dbentry; + PgStat_StatSubWorkerEntry *wentry; + PgStat_StatSubWorkerPreparedXact key; + + prepared_txn = pgstat_get_subworker_prepared_txn(msg->m_subid, + msg->m_gid, true); + Assert(prepared_txn); + switch(msg->m_command) + { + case LOGICAL_REP_MSG_PREPARE: + case LOGICAL_REP_MSG_STREAM_PREPARE: + /* + * Make each size of prepared transaction persistent + * so that we can update stats over the server restart + * and make prepared stats updated when commit prepared + * or rollback prepared arrives. + */ + prepared_txn->subid = msg->m_subid; + strlcpy(prepared_txn->gid, msg->m_gid, sizeof(prepared_txn->gid)); + prepared_txn->xact_size = msg->m_xact_bytes; + break; + + case LOGICAL_REP_MSG_COMMIT_PREPARED: + case LOGICAL_REP_MSG_ROLLBACK_PREPARED: + /* Update exported xact stats now */ + dbentry = pgstat_get_db_entry(msg->m_databaseid, true); + wentry = pgstat_get_subworker_entry(dbentry, + msg->m_subid, + InvalidOid /* apply worker */, + true); + Assert(wentry); + if (msg->m_command == LOGICAL_REP_MSG_COMMIT_PREPARED) + { + wentry->xact_commit_count++; + wentry->xact_commit_bytes += prepared_txn->xact_size; + } + else + { + wentry->xact_abort_count++; + wentry->xact_abort_bytes += prepared_txn->xact_size; + } + + /* Clean up this gid from transaction size hash */ + key.subid = prepared_txn->subid; + strlcpy(key.gid, msg->m_gid, sizeof(key.gid)); + (void) hash_search(subWorkerPreparedXactSizeHash, + (void *) &key, HASH_REMOVE, NULL); + break; + + default: + elog(ERROR, "unexpected logical message type as prepare transaction"); + break; + } +} + +/* ---------- * pgstat_recv_subworker_error() - * * Process a SUBWORKERERROR message. @@ -6166,6 +6405,10 @@ pgstat_recv_subworker_error(PgStat_MsgSubWorkerError *msg, int len) msg->m_subrelid, true); Assert(subwentry); + /* general transaction stats for error */ + subwentry->xact_error_count++; + subwentry->xact_error_bytes += msg->m_xact_error_bytes; + /* * Update only the counter and timestamp if we received the same error * again @@ -6335,6 +6578,12 @@ pgstat_get_subworker_entry(PgStat_StatDBEntry *dbentry, Oid subid, Oid subrelid, /* If not found, initialize the new one */ if (create && !found) { + subwentry->xact_commit_count = 0; + subwentry->xact_commit_bytes = 0; + subwentry->xact_error_count = 0; + subwentry->xact_error_bytes = 0; + subwentry->xact_abort_count = 0; + subwentry->xact_abort_bytes = 0; subwentry->relid = InvalidOid; subwentry->command = 0; subwentry->xid = InvalidTransactionId; @@ -6451,3 +6700,48 @@ pgstat_count_slru_truncate(int slru_idx) { slru_entry(slru_idx)->m_truncate += 1; } + + /* ---------- + * pgstat_get_subworker_prepared_txn + * + * Return subscription worker entry with the given subscription OID and + * gid. + * ---------- + */ +static PgStat_StatSubWorkerPreparedXactSize* +pgstat_get_subworker_prepared_txn(Oid subid, char *gid, bool create) +{ + PgStat_StatSubWorkerPreparedXact key; + PgStat_StatSubWorkerPreparedXactSize *prepared_txn_size; + HASHACTION action; + bool found; + + if (subWorkerPreparedXactSizeHash == NULL) + { + HASHCTL hash_ctl; + + hash_ctl.keysize = sizeof(PgStat_StatSubWorkerPreparedXact); + hash_ctl.entrysize = sizeof(PgStat_StatSubWorkerPreparedXactSize); + hash_ctl.hcxt = pgStatLocalContext; + subWorkerPreparedXactSizeHash = hash_create("Subscription worker stats of prepared txn", + PGSTAT_SUBWORKER_HASH_SIZE, + &hash_ctl, + HASH_ELEM | HASH_STRINGS); + } + + key.subid = subid; + memcpy(key.gid, gid, sizeof(key.gid)); + action = (create ? HASH_ENTER : HASH_FIND); + prepared_txn_size = (PgStat_StatSubWorkerPreparedXactSize *) hash_search(subWorkerPreparedXactSizeHash, + (void *) &key, + action, &found); + + if (create && !found) + { + prepared_txn_size->subid = 0; + prepared_txn_size->gid[0] = '\0'; + prepared_txn_size->xact_size = 0; + } + + return prepared_txn_size; +} diff --git a/src/backend/replication/logical/tablesync.c b/src/backend/replication/logical/tablesync.c index f07983a..aff78fd 100644 --- a/src/backend/replication/logical/tablesync.c +++ b/src/backend/replication/logical/tablesync.c @@ -1149,6 +1149,12 @@ copy_table_done: MyLogicalRepWorker->relstate_lsn = *origin_startpos; SpinLockRelease(&MyLogicalRepWorker->relmutex); + /* Report the success of table sync. */ + pgstat_report_subworker_xact_end(MyLogicalRepWorker->subid, + MyLogicalRepWorker->relid, + 0 /* no logical message type */, + 0 /* xact size */); + /* * Finally, wait until the main apply worker tells us to catch up and then * return to let LogicalRepApplyLoop do it. diff --git a/src/backend/replication/logical/worker.c b/src/backend/replication/logical/worker.c index e2a929b..db7abfc 100644 --- a/src/backend/replication/logical/worker.c +++ b/src/backend/replication/logical/worker.c @@ -221,21 +221,65 @@ typedef struct ApplyErrorCallbackArg LogicalRepMsgType command; /* 0 if invalid */ LogicalRepRelMapEntry *rel; + /* + * Store data size of this transaction. + * + * The byte size of transaction on the publisher is calculated + * by ReorderBufferChangeSize() based on the ReorderBufferChange + * structure. But on the subscriber, consumed resources are + * not same as the publisher's decoding processsing and required + * to be computed in different way. Therefore, the exact same byte + * size is not restored on the subscriber usually. + * + * Data size of streaming transactions is managed by streamingXactSize + * for flexible data blocks handling. + */ + PgStat_Counter bytes; + /* Remote node information */ int remote_attnum; /* -1 if invalid */ TransactionId remote_xid; TimestampTz ts; /* commit, rollback, or prepare timestamp */ } ApplyErrorCallbackArg; +/* Struct to indicate extra consumption of transaction size */ +typedef union ApplyTxnExtraData +{ + LogicalRepRelMapEntry *relmapentry; + LogicalRepRelation *reprelation; + int *stream_write_len; +} ApplyTxnExtraData; + static ApplyErrorCallbackArg apply_error_callback_arg = { .command = 0, .rel = NULL, + .bytes = 0, .remote_attnum = -1, .remote_xid = InvalidTransactionId, .ts = 0, }; +/* + * Two or more streaming transactions in parallel on the publisher + * generate unexpected order of partial txn data demarcated stream start + * and stream stop to the subscriber, since whenever its size of one of + * the txns reaches the publisher's logical_decoding_work_mem, + * the part (and in the end, last remaining changes) is streamed. + * This creates mixed blocks of streaming data while + * there's possibility some are successfully committed but others are + * not by stream abort. Therefore, to track correct byte size + * it's necessary to trace each streaming transaction by making pair + * of xid and transaction size. + */ +#define PARALLEL_STREAMING_XACTS 32 +typedef struct XactSizeEntry +{ + TransactionId key; + PgStat_Counter xact_size; +} XactSizeEntry; +static HTAB *streamingXactSize = NULL; + static MemoryContext ApplyMessageContext = NULL; MemoryContext ApplyContext = NULL; @@ -304,6 +348,9 @@ static void maybe_reread_subscription(void); /* prototype needed because of stream_commit */ static void apply_dispatch(StringInfo s); +static void update_apply_change_size(LogicalRepMsgType action, + ApplyTxnExtraData *extra_data); + static void apply_handle_commit_internal(LogicalRepCommitData *commit_data); static void apply_handle_insert_internal(ApplyExecutionData *edata, ResultRelInfo *relinfo, @@ -818,6 +865,11 @@ apply_handle_commit(StringInfo s) /* Process any tables that are being synchronized in parallel. */ process_syncing_tables(commit_data.end_lsn); + pgstat_report_subworker_xact_end(MyLogicalRepWorker->subid, + InvalidOid, + LOGICAL_REP_MSG_COMMIT, + get_apply_error_context_xact_size()); + pgstat_report_activity(STATE_IDLE, NULL); reset_apply_error_context_info(); } @@ -922,6 +974,11 @@ apply_handle_prepare(StringInfo s) /* Process any tables that are being synchronized in parallel. */ process_syncing_tables(prepare_data.end_lsn); + pgstat_report_subworker_twophase_xact(MyLogicalRepWorker->subid, + LOGICAL_REP_MSG_PREPARE, + get_apply_error_context_xact_size(), + prepare_data.gid); + pgstat_report_activity(STATE_IDLE, NULL); reset_apply_error_context_info(); } @@ -963,6 +1020,13 @@ apply_handle_commit_prepared(StringInfo s) /* Process any tables that are being synchronized in parallel. */ process_syncing_tables(prepare_data.end_lsn); + update_apply_change_size(LOGICAL_REP_MSG_COMMIT_PREPARED, NULL); + + pgstat_report_subworker_twophase_xact(MyLogicalRepWorker->subid, + LOGICAL_REP_MSG_COMMIT_PREPARED, + get_apply_error_context_xact_size(), + prepare_data.gid); + pgstat_report_activity(STATE_IDLE, NULL); reset_apply_error_context_info(); } @@ -1014,6 +1078,12 @@ apply_handle_rollback_prepared(StringInfo s) /* Process any tables that are being synchronized in parallel. */ process_syncing_tables(rollback_data.rollback_end_lsn); + /* send rollback prepared message for this gid */ + pgstat_report_subworker_twophase_xact(MyLogicalRepWorker->subid, + LOGICAL_REP_MSG_ROLLBACK_PREPARED, + get_apply_error_context_xact_size(), + rollback_data.gid); + pgstat_report_activity(STATE_IDLE, NULL); reset_apply_error_context_info(); } @@ -1029,6 +1099,7 @@ static void apply_handle_stream_prepare(StringInfo s) { LogicalRepPreparedTxnData prepare_data; + XactSizeEntry *streamed_entry; if (in_streamed_transaction) ereport(ERROR, @@ -1066,6 +1137,19 @@ apply_handle_stream_prepare(StringInfo s) /* Process any tables that are being synchronized in parallel. */ process_syncing_tables(prepare_data.end_lsn); + /* + * Report the prepared streaming xact size to the stats collector + * in a prepared xact manner to make it survive over the restart. + */ + streamed_entry = hash_search(streamingXactSize, + (void *) &prepare_data.xid, + HASH_FIND, NULL); + Assert(streamed_entry); + pgstat_report_subworker_twophase_xact(MyLogicalRepWorker->subid, + LOGICAL_REP_MSG_STREAM_PREPARE, + streamed_entry->xact_size, + prepare_data.gid); + pgstat_report_activity(STATE_IDLE, NULL); reset_apply_error_context_info(); @@ -1143,6 +1227,8 @@ apply_handle_stream_start(StringInfo s) MyLogicalRepWorker->stream_fileset = palloc(sizeof(FileSet)); FileSetInit(MyLogicalRepWorker->stream_fileset); + update_apply_change_size(LOGICAL_REP_MSG_STREAM_START, NULL); + MemoryContextSwitchTo(oldctx); } @@ -1193,12 +1279,17 @@ apply_handle_stream_stop(StringInfo s) /* * Handle STREAM abort message. + * + * Currently, abort of streaming subtransaction does not affect + * size of streaming transaction resources because it has used the + * resources anyway. */ static void apply_handle_stream_abort(StringInfo s) { TransactionId xid; TransactionId subxid; + XactSizeEntry *streamed_entry; if (in_streamed_transaction) ereport(ERROR, @@ -1213,8 +1304,36 @@ apply_handle_stream_abort(StringInfo s) */ if (xid == subxid) { + bool found = false; set_apply_error_context_xact(xid, 0); stream_cleanup_files(MyLogicalRepWorker->subid, xid); + + /* + * We've completed to handle stream abort without issue, so + * get ready to report the transaction stats via normal + * termination route instead of the apply error route. + */ + streamed_entry = hash_search(streamingXactSize, + (void *) &xid, + HASH_FIND, &found); + /* + * It's possible that we get stream abort + * earlier than any call of write_stream_change that + * creates one hash entry for this xid. In this case, + * to find a entry with this xid fails. So just check + * if we've found it. Only when we confirm some writes + * by write_stream_change, report the stream_abort. + */ + if (found) + { + pgstat_report_subworker_xact_end(MyLogicalRepWorker->subid, + InvalidOid, + LOGICAL_REP_MSG_STREAM_ABORT, + streamed_entry->xact_size); + (void) hash_search(streamingXactSize, + (void *) &xid, + HASH_REMOVE, NULL); + } } else { @@ -1417,6 +1536,7 @@ apply_handle_stream_commit(StringInfo s) { TransactionId xid; LogicalRepCommitData commit_data; + XactSizeEntry *streamed_entry; if (in_streamed_transaction) ereport(ERROR, @@ -1438,6 +1558,19 @@ apply_handle_stream_commit(StringInfo s) /* Process any tables that are being synchronized in parallel. */ process_syncing_tables(commit_data.end_lsn); + /* Report and clean up the xid */ + streamed_entry = hash_search(streamingXactSize, + (void *) &xid, + HASH_FIND, NULL); + Assert(streamed_entry); + pgstat_report_subworker_xact_end(MyLogicalRepWorker->subid, + InvalidOid, + LOGICAL_REP_MSG_STREAM_COMMIT, + streamed_entry->xact_size); + (void) hash_search(streamingXactSize, + (void *) &xid, + HASH_REMOVE, NULL); + pgstat_report_activity(STATE_IDLE, NULL); reset_apply_error_context_info(); @@ -1541,6 +1674,7 @@ apply_handle_insert(StringInfo s) EState *estate; TupleTableSlot *remoteslot; MemoryContext oldctx; + ApplyTxnExtraData extra_size; if (handle_streamed_transaction(LOGICAL_REP_MSG_INSERT, s)) return; @@ -1576,6 +1710,10 @@ apply_handle_insert(StringInfo s) slot_fill_defaults(rel, estate, remoteslot); MemoryContextSwitchTo(oldctx); + /* Update transaction size */ + extra_size.relmapentry = rel; + update_apply_change_size(LOGICAL_REP_MSG_INSERT, &extra_size); + /* For a partitioned table, insert the tuple into a partition. */ if (rel->localrel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE) apply_handle_tuple_routing(edata, @@ -1667,6 +1805,7 @@ apply_handle_update(StringInfo s) TupleTableSlot *remoteslot; RangeTblEntry *target_rte; MemoryContext oldctx; + ApplyTxnExtraData extra_size; if (handle_streamed_transaction(LOGICAL_REP_MSG_UPDATE, s)) return; @@ -1733,6 +1872,10 @@ apply_handle_update(StringInfo s) has_oldtup ? &oldtup : &newtup); MemoryContextSwitchTo(oldctx); + /* Update transaction size */ + extra_size.relmapentry = rel; + update_apply_change_size(LOGICAL_REP_MSG_UPDATE, &extra_size); + /* For a partitioned table, apply update to correct partition. */ if (rel->localrel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE) apply_handle_tuple_routing(edata, @@ -1830,6 +1973,7 @@ apply_handle_delete(StringInfo s) EState *estate; TupleTableSlot *remoteslot; MemoryContext oldctx; + ApplyTxnExtraData extra_size; if (handle_streamed_transaction(LOGICAL_REP_MSG_DELETE, s)) return; @@ -1867,6 +2011,10 @@ apply_handle_delete(StringInfo s) slot_store_data(remoteslot, rel, &oldtup); MemoryContextSwitchTo(oldctx); + /* Update transaction size */ + extra_size.relmapentry = rel; + update_apply_change_size(LOGICAL_REP_MSG_DELETE, &extra_size); + /* For a partitioned table, apply delete to correct partition. */ if (rel->localrel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE) apply_handle_tuple_routing(edata, @@ -2418,6 +2566,111 @@ apply_dispatch(StringInfo s) } /* + * Subscriber side implementation equivalent to ReorderBufferChangeSize + * of the publisher. + * + * According to the logical replication message type, record major + * resource consumptions of this subscription for each message. + * At present, do not collect data from generic functions to keep + * code simplicity, since the implementation complexity versus benefit + * tradeoff should not be good. Also, add multiple values + * at once in order to reduce the number of calls to this function. + * + * 'extra_data' controls detail handling of data size calculation. + */ +static void +update_apply_change_size(LogicalRepMsgType action, ApplyTxnExtraData *extra_data) +{ + int64 size = 0; + + /* + * In streaming mode, stream_write_change is called + * instead of immediate apply. List up the messages types + * that can be caught by handle_streamed_transaction and + * treat the write length as the size of transaction so + * that we can export it as part of pg_stat_subscription_workers. + */ + if (in_streamed_transaction && + (action == LOGICAL_REP_MSG_INSERT || + action == LOGICAL_REP_MSG_UPDATE || + action == LOGICAL_REP_MSG_DELETE || + action == LOGICAL_REP_MSG_TRUNCATE || + action == LOGICAL_REP_MSG_RELATION || + action == LOGICAL_REP_MSG_TYPE)) + { + size += *extra_data->stream_write_len; + add_apply_error_context_xact_size(size); + return; + } + + switch (action) + { + /* No special memory consumption */ + case LOGICAL_REP_MSG_BEGIN: + case LOGICAL_REP_MSG_COMMIT: + case LOGICAL_REP_MSG_TRUNCATE: + case LOGICAL_REP_MSG_TYPE: + case LOGICAL_REP_MSG_ORIGIN: + case LOGICAL_REP_MSG_MESSAGE: + case LOGICAL_REP_MSG_STREAM_STOP: + case LOGICAL_REP_MSG_STREAM_ABORT: + case LOGICAL_REP_MSG_BEGIN_PREPARE: + break; + + case LOGICAL_REP_MSG_INSERT: + case LOGICAL_REP_MSG_UPDATE: + case LOGICAL_REP_MSG_DELETE: + Assert(extra_data != NULL); + + /* + * Compute size based on ApplyExecutionData. + * The size of LogicalRepRelMapEntry can be skipped because + * it is obtained from hash_search in logicalrep_rel_open. + */ + size += sizeof(ApplyExecutionData) + sizeof(EState) + + sizeof(ResultRelInfo) + sizeof(ResultRelInfo); + + /* + * Add some extra size if the target relation is partitioned. + * PartitionTupleRouting isn't exported. Therefore, call the + * function that returns its size instead. + */ + if (extra_data->relmapentry->localrel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE) + size += sizeof(ModifyTableState) + PartitionTupleRoutingSize(); + break; + + case LOGICAL_REP_MSG_RELATION: + Assert(extra_data != NULL); + + /* See logicalrep_read_attrs for the last two */ + size += sizeof(LogicalRepRelation) + + extra_data->reprelation->natts * sizeof(char *) + + extra_data->reprelation->natts * sizeof(Oid); + break; + + case LOGICAL_REP_MSG_STREAM_START: + size += sizeof(FileSet); + break; + + case LOGICAL_REP_MSG_PREPARE: + case LOGICAL_REP_MSG_COMMIT_PREPARED: + case LOGICAL_REP_MSG_ROLLBACK_PREPARED: + case LOGICAL_REP_MSG_STREAM_COMMIT: + case LOGICAL_REP_MSG_STREAM_PREPARE: + size += sizeof(FlushPosition); + break; + + default: + ereport(ERROR, + (errcode(ERRCODE_PROTOCOL_VIOLATION), + errmsg("invalid logical replication message type \"%c\"", action))); + } + + /* update the total size of consumption */ + add_apply_error_context_xact_size(size); +} + +/* * Figure out which write/flush positions to report to the walsender process. * * We can't simply report back the last LSN the walsender sent us because the @@ -3271,6 +3524,9 @@ static void stream_write_change(char action, StringInfo s) { int len; + int total_len; + bool found; + XactSizeEntry *streamed_entry; Assert(in_streamed_transaction); Assert(TransactionIdIsValid(stream_xid)); @@ -3289,6 +3545,16 @@ stream_write_change(char action, StringInfo s) len = (s->len - s->cursor); BufFileWrite(stream_fd, &s->data[s->cursor], len); + + /* update xact size by xid */ + total_len = (s->len - s->cursor) * 2 + sizeof(char) + sizeof(action); + streamed_entry = (XactSizeEntry *) hash_search(streamingXactSize, + (void *) &stream_xid, + HASH_ENTER, &found); + if (!found) + streamed_entry->xact_size = total_len; /* init */ + else + streamed_entry->xact_size += total_len; /* update */ } /* @@ -3426,6 +3692,23 @@ ApplyWorkerMain(Datum main_arg) elog(DEBUG1, "connecting to publisher using connection string \"%s\"", MySubscription->conninfo); + /* + * Initialize the apply worker's hash to manage bytes for + * streaming txns. + */ + if (!am_tablesync_worker() && MySubscription->stream) + { + HASHCTL hash_ctl; + + hash_ctl.keysize = sizeof(TransactionId); + hash_ctl.entrysize = sizeof(XactSizeEntry); + hash_ctl.hcxt = LogicalStreamingContext; + streamingXactSize = hash_create("xact size per streaming xid", + PARALLEL_STREAMING_XACTS, + &hash_ctl, + HASH_ELEM | HASH_BLOBS | HASH_CONTEXT); + } + if (am_tablesync_worker()) { char *syncslotname; @@ -3601,6 +3884,27 @@ ApplyWorkerMain(Datum main_arg) proc_exit(0); } +/* Exported so that stats collector can utilize this value */ +int64 +get_apply_error_context_xact_size(void) +{ + return apply_error_callback_arg.bytes; +} + +/* Add size to apply error bytes */ +void +add_apply_error_context_xact_size(int64 size) +{ + apply_error_callback_arg.bytes += size; +} + +/* Reset information of apply error callback */ +void +reset_apply_error_context_xact_size(void) +{ + apply_error_callback_arg.bytes = 0; +} + /* * Is current process a logical replication worker? */ diff --git a/src/backend/utils/adt/pgstatfuncs.c b/src/backend/utils/adt/pgstatfuncs.c index aa17b82..c902795 100644 --- a/src/backend/utils/adt/pgstatfuncs.c +++ b/src/backend/utils/adt/pgstatfuncs.c @@ -2411,7 +2411,7 @@ pg_stat_get_replication_slot(PG_FUNCTION_ARGS) Datum pg_stat_get_subscription_worker(PG_FUNCTION_ARGS) { -#define PG_STAT_GET_SUBSCRIPTION_WORKER_COLS 8 +#define PG_STAT_GET_SUBSCRIPTION_WORKER_COLS 14 Oid subid = PG_GETARG_OID(0); Oid subrelid; TupleDesc tupdesc; @@ -2438,17 +2438,29 @@ pg_stat_get_subscription_worker(PG_FUNCTION_ARGS) OIDOID, -1, 0); TupleDescInitEntry(tupdesc, (AttrNumber) 2, "subrelid", OIDOID, -1, 0); - TupleDescInitEntry(tupdesc, (AttrNumber) 3, "relid", + TupleDescInitEntry(tupdesc, (AttrNumber) 3, "commit_count", + INT8OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) 4, "commit_bytes", + INT8OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) 5, "error_count", + INT8OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) 6, "error_bytes", + INT8OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) 7, "abort_count", + INT8OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) 8, "abort_bytes", + INT8OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) 9, "last_error_relid", OIDOID, -1, 0); - TupleDescInitEntry(tupdesc, (AttrNumber) 4, "command", + TupleDescInitEntry(tupdesc, (AttrNumber) 10, "last_error_command", TEXTOID, -1, 0); - TupleDescInitEntry(tupdesc, (AttrNumber) 5, "xid", + TupleDescInitEntry(tupdesc, (AttrNumber) 11, "last_error_xid", XIDOID, -1, 0); - TupleDescInitEntry(tupdesc, (AttrNumber) 6, "error_count", + TupleDescInitEntry(tupdesc, (AttrNumber) 12, "last_error_count", INT8OID, -1, 0); - TupleDescInitEntry(tupdesc, (AttrNumber) 7, "error_message", + TupleDescInitEntry(tupdesc, (AttrNumber) 13, "last_error_message", TEXTOID, -1, 0); - TupleDescInitEntry(tupdesc, (AttrNumber) 8, "last_error_time", + TupleDescInitEntry(tupdesc, (AttrNumber) 14, "last_error_time", TIMESTAMPTZOID, -1, 0); BlessTupleDesc(tupdesc); @@ -2466,28 +2478,36 @@ pg_stat_get_subscription_worker(PG_FUNCTION_ARGS) else nulls[i++] = true; - /* relid */ + /* transaction stats */ + values[i++] = Int64GetDatum(wentry->xact_commit_count); + values[i++] = Int64GetDatum(wentry->xact_commit_bytes); + values[i++] = Int64GetDatum(wentry->xact_error_count); + values[i++] = Int64GetDatum(wentry->xact_error_bytes); + values[i++] = Int64GetDatum(wentry->xact_abort_count); + values[i++] = Int64GetDatum(wentry->xact_abort_bytes); + + /* last_error_relid */ if (OidIsValid(wentry->relid)) values[i++] = ObjectIdGetDatum(wentry->relid); else nulls[i++] = true; - /* command */ + /* last_error_command */ if (wentry->command != 0) values[i++] = CStringGetTextDatum(logicalrep_message_type(wentry->command)); else nulls[i++] = true; - /* xid */ + /* last_error_xid */ if (TransactionIdIsValid(wentry->xid)) values[i++] = TransactionIdGetDatum(wentry->xid); else nulls[i++] = true; - /* error_count */ + /* last_error_count */ values[i++] = Int64GetDatum(wentry->error_count); - /* error_message */ + /* last_error_message */ values[i++] = CStringGetTextDatum(wentry->error_message); /* last_error_time */ diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index 528c39d..f276b0b 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -5389,9 +5389,9 @@ proname => 'pg_stat_get_subscription_worker', prorows => '1', proisstrict => 'f', proretset => 't', provolatile => 's', proparallel => 'r', prorettype => 'record', proargtypes => 'oid oid', - proallargtypes => '{oid,oid,oid,oid,oid,text,xid,int8,text,timestamptz}', - proargmodes => '{i,i,o,o,o,o,o,o,o,o}', - proargnames => '{subid,subrelid,subid,subrelid,relid,command,xid,error_count,error_message,last_error_time}', + proallargtypes => '{oid,oid,oid,oid,int8,int8,int8,int8,int8,int8,oid,text,xid,int8,text,timestamptz}', + proargmodes => '{i,i,o,o,o,o,o,o,o,o,o,o,o,o,o,o}', + proargnames => '{subid,subrelid,subid,subrelid,commit_count,commit_bytes,error_count,error_bytes,abort_count,abort_bytes,last_error_relid,last_error_command,last_error_xid,last_error_count,last_error_message,last_error_time}', prosrc => 'pg_stat_get_subscription_worker' }, { oid => '6118', descr => 'statistics: information about subscription', proname => 'pg_stat_get_subscription', prorows => '10', proisstrict => 'f', diff --git a/src/include/executor/execPartition.h b/src/include/executor/execPartition.h index 694e38b..773e46c 100644 --- a/src/include/executor/execPartition.h +++ b/src/include/executor/execPartition.h @@ -110,6 +110,7 @@ typedef struct PartitionPruneState PartitionPruningData *partprunedata[FLEXIBLE_ARRAY_MEMBER]; } PartitionPruneState; +extern size_t PartitionTupleRoutingSize(void); extern PartitionTupleRouting *ExecSetupPartitionTupleRouting(EState *estate, Relation rel); extern ResultRelInfo *ExecFindPartition(ModifyTableState *mtstate, diff --git a/src/include/pgstat.h b/src/include/pgstat.h index 3749bac..91a92da 100644 --- a/src/include/pgstat.h +++ b/src/include/pgstat.h @@ -87,6 +87,8 @@ typedef enum StatMsgType PGSTAT_MTYPE_DISCONNECT, PGSTAT_MTYPE_SUBSCRIPTIONPURGE, PGSTAT_MTYPE_SUBWORKERERROR, + PGSTAT_MTYPE_SUBWORKERXACTEND, + PGSTAT_MTYPE_SUBWORKERTWOPHASEXACT, } StatMsgType; /* ---------- @@ -579,6 +581,56 @@ typedef struct PgStat_MsgSubscriptionPurge } PgStat_MsgSubscriptionPurge; /* ---------- + * PgStat_MsgSubscriptionXactEnd Sent by the apply worker or the table sync worker + * to report successful transaction ends. + * ---------- + */ +typedef struct PgStat_MsgSubWorkerXactEnd +{ + PgStat_MsgHdr m_hdr; + + /* determine the worker entry */ + Oid m_databaseid; + Oid m_subid; + Oid m_subrelid; + + /* + * distinguish between transaction commits and streaming transaction aborts + * that are handled without error. + */ + LogicalRepMsgType m_command; + + /* memory consumption used by transaction */ + PgStat_Counter m_xact_bytes; + +} PgStat_MsgSubWorkerXactEnd; + +/* ---------- + * PgStat_MsgSubWorkerTwophaseXact Sent by the apply worker to make size of prepared + * txn persistent over the server restart and make it + * visible after commit prepare or rollback prepared. + * This is separated from PgStat_MsgSubWorkerXactEnd + * so that we can reduce message size of gid for other + * operations (e.g. normal COMMIT) that should happen more + * frequently than prepare operation usually. + * ---------- + */ +typedef struct PgStat_MsgSubWorkerTwophaseXact +{ + PgStat_MsgHdr m_hdr; + + /* determine the subscription */ + Oid m_databaseid; + Oid m_subid; + + LogicalRepMsgType m_command; + char m_gid[GIDSIZE]; + int gid_len; + PgStat_Counter m_xact_bytes; + +} PgStat_MsgSubWorkerTwophaseXact; + +/* ---------- * PgStat_MsgSubWorkerError Sent by the apply worker or the table sync worker to * report the error occurred during logical replication. * ---------- @@ -598,6 +650,12 @@ typedef struct PgStat_MsgSubWorkerError Oid m_subrelid; /* + * Transaction stats of subscription needs to be updated when an + * error occurs. + */ + PgStat_Counter m_xact_error_bytes; + + /* * Oids of the database and the table that the reporter was actually * processing. m_relid can be InvalidOid if an error occurred during * worker applying a non-data-modification message such as RELATION. @@ -790,6 +848,8 @@ typedef union PgStat_Msg PgStat_MsgDisconnect msg_disconnect; PgStat_MsgSubscriptionPurge msg_subscriptionpurge; PgStat_MsgSubWorkerError msg_subworkererror; + PgStat_MsgSubWorkerXactEnd msg_subworkerxactend; + PgStat_MsgSubWorkerTwophaseXact msg_subworkertwophasexact; } PgStat_Msg; @@ -1027,6 +1087,16 @@ typedef struct PgStat_StatSubWorkerEntry PgStat_StatSubWorkerKey key; /* hash key (must be first) */ /* + * Cumulative transaction statistics of subscription worker + */ + PgStat_Counter xact_commit_count; + PgStat_Counter xact_commit_bytes; + PgStat_Counter xact_error_count; + PgStat_Counter xact_error_bytes; + PgStat_Counter xact_abort_count; + PgStat_Counter xact_abort_bytes; + + /* * Subscription worker error statistics representing an error that * occurred during application of logical replication or the initial table * synchronization. @@ -1040,6 +1110,22 @@ typedef struct PgStat_StatSubWorkerEntry char error_message[PGSTAT_SUBWORKERERROR_MSGLEN]; } PgStat_StatSubWorkerEntry; +/* prepared transaction */ +typedef struct PgStat_StatSubWorkerPreparedXact +{ + Oid subid; + char gid[GIDSIZE]; +} PgStat_StatSubWorkerPreparedXact; + +typedef struct PgStat_StatSubWorkerPreparedXactSize +{ + PgStat_StatSubWorkerPreparedXact key; /* hash key */ + + Oid subid; + char gid[GIDSIZE]; + PgStat_Counter xact_size; +} PgStat_StatSubWorkerPreparedXactSize; + /* * Working state needed to accumulate per-function-call timing statistics. */ @@ -1149,6 +1235,10 @@ extern void pgstat_report_checksum_failure(void); extern void pgstat_report_replslot(const PgStat_StatReplSlotEntry *repSlotStat); extern void pgstat_report_replslot_create(const char *slotname); extern void pgstat_report_replslot_drop(const char *slotname); +extern void pgstat_report_subworker_xact_end(Oid subid, Oid subrel, + LogicalRepMsgType command, PgStat_Counter xact_size); +extern void pgstat_report_subworker_twophase_xact(Oid subid, LogicalRepMsgType command, + PgStat_Counter xact_size, char *gid); extern void pgstat_report_subworker_error(Oid subid, Oid subrelid, Oid relid, LogicalRepMsgType command, TransactionId xid, const char *errmsg); diff --git a/src/include/replication/logicalworker.h b/src/include/replication/logicalworker.h index 2ad61a0..9a8447b 100644 --- a/src/include/replication/logicalworker.h +++ b/src/include/replication/logicalworker.h @@ -16,4 +16,9 @@ extern void ApplyWorkerMain(Datum main_arg); extern bool IsLogicalWorker(void); +/* for transaction stats */ +extern int64 get_apply_error_context_xact_size(void); +extern void add_apply_error_context_xact_size(int64 size); +extern void reset_apply_error_context_xact_size(void); + #endif /* LOGICALWORKER_H */ diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out index d7d17b7..7445041 100644 --- a/src/test/regress/expected/rules.out +++ b/src/test/regress/expected/rules.out @@ -2097,11 +2097,17 @@ pg_stat_subscription| SELECT su.oid AS subid, pg_stat_subscription_workers| SELECT w.subid, s.subname, w.subrelid, - w.relid, - w.command, - w.xid, + w.commit_count, + w.commit_bytes, w.error_count, - w.error_message, + w.error_bytes, + w.abort_count, + w.abort_bytes, + w.last_error_relid, + w.last_error_command, + w.last_error_xid, + w.last_error_count, + w.last_error_message, w.last_error_time FROM ( SELECT pg_subscription.oid AS subid, NULL::oid AS relid @@ -2111,7 +2117,7 @@ pg_stat_subscription_workers| SELECT w.subid, pg_subscription_rel.srrelid AS relid FROM pg_subscription_rel WHERE (pg_subscription_rel.srsubstate <> 'r'::"char")) sr, - (LATERAL pg_stat_get_subscription_worker(sr.subid, sr.relid) w(subid, subrelid, relid, command, xid, error_count, error_message, last_error_time) + (LATERAL pg_stat_get_subscription_worker(sr.subid, sr.relid) w(subid, subrelid, commit_count, commit_bytes, error_count, error_bytes, abort_count, abort_bytes, last_error_relid, last_error_command, last_error_xid, last_error_count, last_error_message, last_error_time) JOIN pg_subscription s ON ((w.subid = s.oid))); pg_stat_sys_indexes| SELECT pg_stat_all_indexes.relid, pg_stat_all_indexes.indexrelid, diff --git a/src/test/subscription/t/026_error_report.pl b/src/test/subscription/t/026_error_report.pl index 3d23bb5..9ff619e 100644 --- a/src/test/subscription/t/026_error_report.pl +++ b/src/test/subscription/t/026_error_report.pl @@ -15,8 +15,8 @@ sub test_subscription_error my $check_sql = qq[ SELECT count(1) > 0 FROM pg_stat_subscription_workers -WHERE relid = '$relname'::regclass]; - $check_sql .= " AND xid = '$xid'::xid;" if $xid ne ''; +WHERE last_error_relid = '$relname'::regclass]; + $check_sql .= " AND last_error_xid = '$xid'::xid;" if $xid ne ''; # Wait for the error statistics to be updated. $node->poll_query_until( @@ -26,9 +26,9 @@ WHERE relid = '$relname'::regclass]; my $result = $node->safe_psql( 'postgres', qq[ -SELECT subname, command, relid::regclass, error_count > 0 +SELECT subname, last_error_command, last_error_relid::regclass, last_error_count > 0 FROM pg_stat_subscription_workers -WHERE relid = '$relname'::regclass; +WHERE last_error_relid = '$relname'::regclass; ]); is($result, $expected_error, $msg); } diff --git a/src/test/subscription/t/027_worker_xact_stats.pl b/src/test/subscription/t/027_worker_xact_stats.pl new file mode 100644 index 0000000..0057e8d --- /dev/null +++ b/src/test/subscription/t/027_worker_xact_stats.pl @@ -0,0 +1,173 @@ + +# Copyright (c) 2021, PostgreSQL Global Development Group + +# Tests for subscription worker statistics during apply. +use strict; +use warnings; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More tests => 2; + +# Create publisher node +my $node_publisher = PostgreSQL::Test::Cluster->new('publisher'); +$node_publisher->init(allows_streaming => 'logical'); +$node_publisher->append_conf('postgresql.conf', qq[ +logical_decoding_work_mem = 64kB +max_prepared_transactions = 10 +max_wal_senders = 10 +wal_sender_timeout = 0 +]); +$node_publisher->start; + +# Create subscriber node +my $node_subscriber = PostgreSQL::Test::Cluster->new('subscriber'); +$node_subscriber->init(allows_streaming => 'logical'); +$node_subscriber->append_conf('postgresql.conf', qq[ +max_prepared_transactions = 10 +log_min_messages = DEBUG1 +]); +$node_subscriber->start; + +# Setup structures on the publisher and the subscriber +$node_publisher->safe_psql('postgres', + "CREATE TABLE test_tab (a int)"); +$node_subscriber->safe_psql('postgres', + "CREATE TABLE test_tab (a int primary key);"); + +# Setup logical replication +my $publisher_connstr = $node_publisher->connstr . ' dbname=postgres'; +$node_publisher->safe_psql('postgres', + "CREATE PUBLICATION tap_pub FOR TABLE test_tab"); + +my $appname = 'tap_sub'; +my $subopts = 'streaming = on, two_phase = on, copy_data = false'; +$node_subscriber->safe_psql('postgres', qq[ +CREATE SUBSCRIPTION tap_sub +CONNECTION '$publisher_connstr application_name=$appname' +PUBLICATION tap_pub WITH ($subopts); +]); + +$node_publisher->wait_for_catchup($appname); + +# There's no entry at the beginning +my $result = $node_subscriber->safe_psql('postgres', +"SELECT count(*) FROM pg_stat_subscription_workers;"); +is($result, q(0), 'no entry for transaction stats yet'); + +# COMMIT +$node_publisher->safe_psql('postgres', + "BEGIN; INSERT INTO test_tab VALUES (1); COMMIT;"); + +$node_subscriber->poll_query_until('postgres', +"SELECT count(1) = 1 FROM pg_stat_subscription_workers where commit_count = 1;") + or die "didn't get updates of xact stats by commit"; + +# Now, stats collector make the bytes updated also. +$result = $node_subscriber->safe_psql('postgres', +"SELECT commit_bytes > 0 FROM pg_stat_subscription_workers;"); +is($result, q(t), 'got consumed bytes'); + +# STREAM COMMIT +$node_publisher->safe_psql( + 'postgres', + "BEGIN; INSERT INTO test_tab VALUES(generate_series(1001, 2000)); COMMIT;"); +$node_subscriber->poll_query_until('postgres', +"SELECT count(1) = 1 FROM pg_stat_subscription_workers where commit_count = 2;") + or die "didn't get updates of xact stats by stream commit"; + +# STREAM PREPARE & COMMIT PREPARED +# This should restore the xact size before the shutdown. +$node_publisher->safe_psql( + 'postgres', q[ +BEGIN; +INSERT INTO test_tab VALUES(generate_series(2001, 3000)); +PREPARE TRANSACTION 'gid1'; +]); + +# This streamed prepare is not displayed until the commit prepared +# or rollback prepared. Hence, there's no way to confirm that +# stats collector has received the bytes of prepared transaction. +# So, instead of checking the view, issue one more committed transaction +# after the prepare and make sure that this commit's update is done, +# which should mean the previous streamed prepare is already processed +# by the stats collector as well. +$node_publisher->safe_psql('postgres', +"BEGIN; INSERT INTO test_tab VALUES (2); COMMIT;"); +$node_subscriber->poll_query_until('postgres', +"SELECT count(1) = 1 FROM pg_stat_subscription_workers where commit_count = 3;") + or die "didn't process the updates of committed transaction"; + +$node_subscriber->restart; + +# Get commit_bytes before commit prepared. +my $tmp = $node_subscriber->safe_psql('postgres', +"SELECT commit_bytes FROM pg_stat_subscription_workers where commit_count = 3;"); + +# Commit prepared increments the commit_count. +$node_publisher->safe_psql('postgres', "COMMIT PREPARED 'gid1'"); +$node_subscriber->poll_query_until('postgres', +"SELECT count(1) = 1 FROM pg_stat_subscription_workers where commit_count = 4;") + or die "didn't get updates of xact stats by stream prepare and commit prepared"; + +$node_subscriber->poll_query_until('postgres', +"SELECT commit_bytes > $tmp FROM pg_stat_subscription_workers where commit_count = 4;"); + +# STREAM ABORT +# Store previous stream_counts to recognize +# another streaming from the increase of this value below. +$tmp = $node_publisher->safe_psql('postgres', +"SELECT stream_count FROM pg_stat_replication_slots"); + +# Cause a new streaming and check it by another session +my $in = ''; +my $out = ''; +my $timer = IPC::Run::timeout(180); +my $h = $node_publisher->background_psql('postgres', \$in, \$out, $timer, + on_error_stop => 1); + +$in .= q{ +BEGIN; +INSERT INTO test_tab VALUES(generate_series(3001, 4000)); +}; +$h->pump_nb; + +# Wait until this transaction is streamed certainly +# and after that rollback to send stream abort. +$node_publisher->poll_query_until('postgres', +"SELECT stream_count > $tmp FROM pg_stat_replication_slots;") + or die "didn't stream data of a new transaction"; + +$in .= q{ +ROLLBACK; +\q +}; +$h->finish; + +$node_subscriber->poll_query_until('postgres', +"SELECT count(1) = 1 FROM pg_stat_subscription_workers where abort_count = 1;") + or die "didn't get updates of xact stats by stream abort"; + +# ROLLBACK PREPARED +$node_publisher->safe_psql('postgres', q[ +BEGIN; +INSERT INTO test_tab VALUES (3); +PREPARE TRANSACTION 'gid2'; +]); +$node_publisher->safe_psql('postgres', "ROLLBACK PREPARED 'gid2'"); +$node_subscriber->poll_query_until('postgres', +"SELECT count(1) = 1 FROM pg_stat_subscription_workers where abort_count = 2;") + or die "didn't get updates of xact stats by rollback prepared"; + +# error stats (by duplication error) +$node_publisher->safe_psql('postgres', q[ +BEGIN; +INSERT INTO test_tab VALUES (1); +COMMIT; +]); + +$node_subscriber->poll_query_until('postgres', +"SELECT count(1) = 1 FROM pg_stat_subscription_workers where error_count > 0;") + or die "didn't get updates of xact stats by error"; + +$node_subscriber->stop('fast'); +$node_publisher->stop('fast'); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index e4d78a9..693aae5 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -116,6 +116,7 @@ AppendState ApplyErrorCallbackArg ApplyExecutionData ApplySubXactData +ApplyTxnExtraData Archive ArchiveEntryPtrType ArchiveFormat @@ -1947,6 +1948,8 @@ PgStat_MsgSLRU PgStat_MsgSubWorkerError PgStat_MsgSubWorkerErrorPurge PgStat_MsgSubWorkerPurge +PgStat_MsgSubWorkerTwophaseXact +PgStat_MsgSubWorkerXactEnd PgStat_MsgTabpurge PgStat_MsgTabstat PgStat_MsgTempFile @@ -1960,6 +1963,8 @@ PgStat_StatFuncEntry PgStat_StatReplSlotEntry PgStat_StatSubWorkerEntry PgStat_StatSubWorkerKey +PgStat_StatSubWorkerPreparedXact +PgStat_StatSubWorkerPreparedXactSize PgStat_StatTabEntry PgStat_SubXactStatus PgStat_TableCounts @@ -2957,6 +2962,7 @@ XactCallback XactCallbackItem XactEvent XactLockTableWaitInfo +XactSizeEntry XidBoundsViolation XidCacheStatus XidCommitStatus -- 2.2.0 ^ permalink raw reply [nested|flat] 113+ messages in thread
* RE: Failed transaction statistics to measure the logical replication progress @ 2021-11-09 11:39 [email protected] <[email protected]> parent: vignesh C <[email protected]> 0 siblings, 0 replies; 113+ messages in thread From: [email protected] @ 2021-11-09 11:39 UTC (permalink / raw) To: 'vignesh C' <[email protected]>; +Cc: Greg Nancarrow <[email protected]>; Masahiko Sawada <[email protected]>; Amit Kapila <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]> On Monday, November 8, 2021 3:12 PM vignesh C <[email protected]> wrote: > On Fri, Nov 5, 2021 at 1:42 PM [email protected] > <[email protected]> wrote: > > Lastly, I removed one unnecessary test that checked publisher's stats > > in the TAP tests. > > Also I introduced ApplyTxnExtraData structure to remove void* argument > > of update_apply_change_size that might worsen the readability of codes > > in the previous version. > > Thanks for the updated patch. Thanks you for checking my patch ! > Few comments: > 1) You could remove LogicalRepPreparedTxnData, > LogicalRepCommitPreparedTxnData & LogicalRepRollbackPreparedTxnData > and change it to char *gid to reduce the function parameter and simiplify the > assignment: > + */ > +void > +pgstat_report_subworker_twophase_xact(Oid subid, LogicalRepMsgType > +command, > + > PgStat_Counter xact_size, > + > LogicalRepPreparedTxnData *prepare_data, > + > LogicalRepCommitPreparedTxnData *commit_data, > + > LogicalRepRollbackPreparedTxnData *rollback_data) Fixed. > 2) Shouldn't this change be part of skip xid patch? > - TupleDescInitEntry(tupdesc, (AttrNumber) 4, "command", > + TupleDescInitEntry(tupdesc, (AttrNumber) 10, > + "last_error_command", > TEXTOID, -1, 0); > - TupleDescInitEntry(tupdesc, (AttrNumber) 5, "xid", > + TupleDescInitEntry(tupdesc, (AttrNumber) 11, "last_error_xid", > XIDOID, -1, 0); > - TupleDescInitEntry(tupdesc, (AttrNumber) 6, "error_count", > + TupleDescInitEntry(tupdesc, (AttrNumber) 12, "last_error_count", > INT8OID, -1, 0); > - TupleDescInitEntry(tupdesc, (AttrNumber) 7, "error_message", > + TupleDescInitEntry(tupdesc, (AttrNumber) 13, > + "last_error_message", > TEXTOID, -1, 0); > - TupleDescInitEntry(tupdesc, (AttrNumber) 8, "last_error_time", > + TupleDescInitEntry(tupdesc, (AttrNumber) 14, "last_error_time", Hmm, I didn't think so. Those renames are necessary to make exisiting columns of skip xid separate from newly-introduced xact stats. That means, original names of skip xid columns in v20 by itself are fine and the renames are needed only when this patch gets committed. At present, we cannot guarantee that this patch will be committed so I'd like to take care of those renames. > 3) This newly added structures should be added to typedefs.list: > ApplyTxnExtraData > XactSizeEntry > PgStat_MsgSubWorkerXactEnd > PgStat_MsgSubWorkerTwophaseXact > PgStat_StatSubWorkerPreparedXact > PgStat_StatSubWorkerPreparedXactSize Added. > 4) We are not sending the transaction size in case of table sync, is this > intentional, if so I felt we should document this in > pg_stat_subscription_workers > + /* Report the success of table sync. */ > + pgstat_report_subworker_xact_end(MyLogicalRepWorker->subid, > + > MyLogicalRepWorker->relid, > + > 0 /* no logical message type */, > + > 0 /* xact size */); Right. Updated the doc description. I added the description in a way that the bytes stats are only for apply worker. > 5) pg_stat_subscription_workers has a lot of columns, if we can reduce the > column size the readability will improve, like xact_commit_count to > commit_count, xact_commit_bytes to commit_bytes, etc > + w.xact_commit_count, > + w.xact_commit_bytes, > + w.xact_error_count, > + w.xact_error_bytes, > + w.xact_abort_count, > + w.xact_abort_bytes, It makes sense. Those can be somehow redundant. Tentatively, I renamed only columns' names exported to the users. This is because changing internal data structure as well (e.g. removing the PgStat_StatSubWorkerEntry's prefixes) causes duplication name of 'error_count' members and changing such an internal data structure of skip xid part will have a huge impact of other parts. Kindly imagine a case that we add 'last_' prefix to the all error statistics representing an error of the structure. If you aren't satisfied with this change, please let me know. Best Regards, Takamichi Osumi ^ permalink raw reply [nested|flat] 113+ messages in thread
* RE: Failed transaction statistics to measure the logical replication progress @ 2021-11-09 11:55 [email protected] <[email protected]> parent: [email protected] <[email protected]> 2 siblings, 0 replies; 113+ messages in thread From: [email protected] @ 2021-11-09 11:55 UTC (permalink / raw) To: 'Greg Nancarrow' <[email protected]>; +Cc: Masahiko Sawada <[email protected]>; Amit Kapila <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]> On Tuesday, November 9, 2021 8:35 PM I wrote: > Yes. I've rebased and updated the patch, paying attention to this point. > Attached the updated version. Forgot to note one thing. This is based on the skip xid v20 shared in [1] [1] - https://www.postgresql.org/message-id/CAD21AoAT42mhcqeB1jPfRL1%2BEUHbZk8MMY_fBgsyZvJeKNpG%2Bw%40mail... Best Regards, Takamichi Osumi ^ permalink raw reply [nested|flat] 113+ messages in thread
* Re: Failed transaction statistics to measure the logical replication progress @ 2021-11-10 06:43 Dilip Kumar <[email protected]> parent: [email protected] <[email protected]> 2 siblings, 1 reply; 113+ messages in thread From: Dilip Kumar @ 2021-11-10 06:43 UTC (permalink / raw) To: [email protected] <[email protected]>; +Cc: Greg Nancarrow <[email protected]>; Masahiko Sawada <[email protected]>; Amit Kapila <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]> On Tue, Nov 9, 2021 at 5:05 PM [email protected] <[email protected]> wrote: > > On Tuesday, November 9, 2021 12:08 PM Greg Nancarrow <[email protected]> wrote: > > On Fri, Nov 5, 2021 at 7:11 PM [email protected] > > <[email protected]> wrote: > > > > > > > I did a quick scan through the latest v8 patch and noticed the following things: > I appreciate your review ! > I have reviewed some part of the patch and I have a few comments 1. + <row> + <entry role="catalog_table_entry"><para role="column_definition"> + <structfield>error_count</structfield> <type>bigint</type> + </para> + <para> + Number of transactions that failed to be applied by the table + sync worker or main apply worker in this subscription. + </para></entry> + </row> The error_count, should be number of transaction failed to applied? or it should be number of error? 2. + <row> + <entry role="catalog_table_entry"><para role="column_definition"> + <structfield>error_bytes</structfield> <type>bigint</type> + </para> How different is error_bytes from the abort_bytes? 3. + { + size += *extra_data->stream_write_len; + add_apply_error_context_xact_size(size); + return; + } ^ permalink raw reply [nested|flat] 113+ messages in thread
* RE: Failed transaction statistics to measure the logical replication progress @ 2021-11-10 09:12 [email protected] <[email protected]> parent: Dilip Kumar <[email protected]> 0 siblings, 1 reply; 113+ messages in thread From: [email protected] @ 2021-11-10 09:12 UTC (permalink / raw) To: 'Dilip Kumar' <[email protected]>; +Cc: Greg Nancarrow <[email protected]>; Masahiko Sawada <[email protected]>; Amit Kapila <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]> On Wednesday, November 10, 2021 3:43 PM Dilip Kumar <[email protected]> wrote: > On Tue, Nov 9, 2021 at 5:05 PM [email protected] > <[email protected]> wrote: > > On Tuesday, November 9, 2021 12:08 PM Greg Nancarrow > <[email protected]> wrote: > > > On Fri, Nov 5, 2021 at 7:11 PM [email protected] > > > <[email protected]> wrote: > > > > > > > > > > I did a quick scan through the latest v8 patch and noticed the following > things: > > I appreciate your review ! > I have reviewed some part of the patch and I have a few comments I really appreciate your attention and review. > 1. > + <row> > + <entry role="catalog_table_entry"><para role="column_definition"> > + <structfield>error_count</structfield> <type>bigint</type> > + </para> > + <para> > + Number of transactions that failed to be applied by the table > + sync worker or main apply worker in this subscription. > + </para></entry> > + </row> > > The error_count, should be number of transaction failed to applied? or it should > be number of error? I thought those were same and currently it gets incremented when an error of apply occurs. Then, it equals to the number of total error. May I have the case when we get different values between those two ? I can be missing something. > 2. > + <row> > + <entry role="catalog_table_entry"><para role="column_definition"> > + <structfield>error_bytes</structfield> <type>bigint</type> > + </para> > > How different is error_bytes from the abort_bytes? By the error_bytes, you can see the consumed resources that were acquired during apply but the applying processing stopped by some error. On the other hand, abort_bytes displays bytes used for ROLLBACK PREPARED and stream_abort processing. That's what I intended. > 3. > + { > + size += *extra_data->stream_write_len; > + add_apply_error_context_xact_size(size); > + return; > + } > > From apply_handle_insert(), we are calling update_apply_change_size(), and > inside this function we are dereferencing *extra_data->stream_write_len. > Basically, stream_write_len is in integer pointer and the caller hasn't allocated > memory for that and inside update_apply_change_size, we are directly > dereferencing the pointer, how this can be correct. I'm so sorry to make you confused. I'll just delete the top part that handles streaming bytes calculation in the update_apply_change_size(). It's because now that there is a specific structure to recognize each streaming xid and save transaction size there, which makes the top part in question useless. > And I also see that in the > whole patch stream_write_len, is never used as lvalue so without storing > anything into this why are we trying to use this as rvalue here? This is clearly > an issue. As described above, I'll fix this part and related codes mainly streaming related codes in the next version. Best Regards, Takamichi Osumi ^ permalink raw reply [nested|flat] 113+ messages in thread
* Re: Failed transaction statistics to measure the logical replication progress @ 2021-11-10 10:13 vignesh C <[email protected]> parent: [email protected] <[email protected]> 2 siblings, 2 replies; 113+ messages in thread From: vignesh C @ 2021-11-10 10:13 UTC (permalink / raw) To: [email protected] <[email protected]>; +Cc: Greg Nancarrow <[email protected]>; Masahiko Sawada <[email protected]>; Amit Kapila <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]> On Tue, Nov 9, 2021 at 5:05 PM [email protected] <[email protected]> wrote: > Yes. I've rebased and updated the patch, paying attention to this point. > Attached the updated version. Thanks for the updated patch, few comments: 1) you could rename PgStat_StatSubWorkerPreparedXact to PgStat_SW_PreparedXactKey or a simpler name which includes key and similarly change PgStat_StatSubWorkerPreparedXactSize to PgStat_SW_PreparedXactEntry +/* prepared transaction */ +typedef struct PgStat_StatSubWorkerPreparedXact +{ + Oid subid; + char gid[GIDSIZE]; +} PgStat_StatSubWorkerPreparedXact; + +typedef struct PgStat_StatSubWorkerPreparedXactSize +{ + PgStat_StatSubWorkerPreparedXact key; /* hash key */ + + Oid subid; + char gid[GIDSIZE]; + PgStat_Counter xact_size; +} PgStat_StatSubWorkerPreparedXactSize; + 2) You can change prepared_size to sw_prepared_xact_entry or prepared_xact_entry since it is a hash entry with few fields + if (subWorkerPreparedXactSizeHash) + { + PgStat_StatSubWorkerPreparedXactSize *prepared_size; + + hash_seq_init(&hstat, subWorkerPreparedXactSizeHash); + while((prepared_size = (PgStat_StatSubWorkerPreparedXactSize *) hash_seq_search(&hstat)) != NULL) + { + fputc('P', fpout); + rc = fwrite(prepared_size, sizeof(PgStat_StatSubWorkerPreparedXactSize), 1, fpout); + (void) rc; /* we'll check for error with ferror */ + } 3) This need to be indented - w.relid, - w.command, - w.xid, - w.error_count, - w.error_message, - w.last_error_time + w.commit_count, + w.commit_bytes, + w.error_count, + w.error_bytes, + w.abort_count, + w.abort_bytes, + w.last_error_relid, + w.last_error_command, + w.last_error_xid, + w.last_error_count, + w.last_error_message, + w.last_error_time 4) Instead of adding a function to calculate the size, can we move PartitionTupleRouting from c file to the header file and use sizeof at the caller function? +/* + * PartitionTupleRoutingSize - exported to calculate total data size + * of logical replication mesage apply, because this is one of the + * ApplyExecutionData struct members. + */ +size_t +PartitionTupleRoutingSize(void) +{ + return sizeof(PartitionTupleRouting); +} 5) You could run pgindent and pgperltidy for the code and test code to fix the indent issues. + subWorkerPreparedXactSizeHash = hash_create("Subscription worker stats of prepared txn", + PGSTAT_SUBWORKER_HASH_SIZE, + &hash_ctl, + HASH_ELEM | HASH_STRINGS | HASH_CONTEXT); +# There's no entry at the beginning +my $result = $node_subscriber->safe_psql('postgres', +"SELECT count(*) FROM pg_stat_subscription_workers;"); +is($result, q(0), 'no entry for transaction stats yet'); 6) Few places you have used strlcpy and few places you have used memcpy, you can keep it consistent: + msg.m_command = command; + strlcpy(msg.m_gid, gid, sizeof(msg.m_gid)); + msg.m_xact_bytes = xact_size; + key.subid = subid; + memcpy(key.gid, gid, sizeof(key.gid)); + action = (create ? HASH_ENTER : HASH_FIND); Regards, Vignesh ^ permalink raw reply [nested|flat] 113+ messages in thread
* Re: Failed transaction statistics to measure the logical replication progress @ 2021-11-11 12:47 vignesh C <[email protected]> parent: vignesh C <[email protected]> 1 sibling, 1 reply; 113+ messages in thread From: vignesh C @ 2021-11-11 12:47 UTC (permalink / raw) To: [email protected] <[email protected]>; +Cc: Greg Nancarrow <[email protected]>; Masahiko Sawada <[email protected]>; Amit Kapila <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]> On Wed, Nov 10, 2021 at 3:43 PM vignesh C <[email protected]> wrote: > > On Tue, Nov 9, 2021 at 5:05 PM [email protected] > <[email protected]> wrote: > > Yes. I've rebased and updated the patch, paying attention to this point. > > Attached the updated version. > > Thanks for the updated patch, few comments: > 6) Few places you have used strlcpy and few places you have used > memcpy, you can keep it consistent: > + msg.m_command = command; > + strlcpy(msg.m_gid, gid, sizeof(msg.m_gid)); > + msg.m_xact_bytes = xact_size; > > + key.subid = subid; > + memcpy(key.gid, gid, sizeof(key.gid)); > + action = (create ? HASH_ENTER : HASH_FIND); Few more comments: 1) Here the tuple length is not considered in the calculation, else it will always show the fixed size for any size tuple. Ex varchar insert with 1 byte or varchar insert with 100's of bytes. So I feel we should include the tuple length in the calculation. + case LOGICAL_REP_MSG_INSERT: + case LOGICAL_REP_MSG_UPDATE: + case LOGICAL_REP_MSG_DELETE: + Assert(extra_data != NULL); + + /* + * Compute size based on ApplyExecutionData. + * The size of LogicalRepRelMapEntry can be skipped because + * it is obtained from hash_search in logicalrep_rel_open. + */ + size += sizeof(ApplyExecutionData) + sizeof(EState) + + sizeof(ResultRelInfo) + sizeof(ResultRelInfo); + + /* + * Add some extra size if the target relation is partitioned. + * PartitionTupleRouting isn't exported. Therefore, call the + * function that returns its size instead. + */ + if (extra_data->relmapentry->localrel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE) + size += sizeof(ModifyTableState) + PartitionTupleRoutingSize(); + break; 2) Can this be part of PgStat_StatDBEntry, similar to tables, functions and subworkers. It might be more appropriate to have it there instead of having another global variable. + * Stats of prepared transactions should be displayed + * at either commit prepared or rollback prepared time, even when it's + * after the server restart. We have the apply worker send those statistics + * to the stats collector at prepare time and the startup process restore + * those at restart if necessary. + */ +static HTAB *subWorkerPreparedXactSizeHash = NULL; + +/* Regards, Vignesh ^ permalink raw reply [nested|flat] 113+ messages in thread
* RE: Failed transaction statistics to measure the logical replication progress @ 2021-11-15 09:27 [email protected] <[email protected]> parent: vignesh C <[email protected]> 0 siblings, 1 reply; 113+ messages in thread From: [email protected] @ 2021-11-15 09:27 UTC (permalink / raw) To: 'vignesh C' <[email protected]>; +Cc: Greg Nancarrow <[email protected]>; Masahiko Sawada <[email protected]>; Amit Kapila <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]> On Thursday, November 11, 2021 9:47 PM vignesh C <[email protected]> wrote: > Few more comments: > 1) Here the tuple length is not considered in the calculation, else it will always > show the fixed size for any size tuple. Ex varchar insert with 1 byte or varchar > insert with 100's of bytes. So I feel we should include the tuple length in the > calculation. > + case LOGICAL_REP_MSG_INSERT: > + case LOGICAL_REP_MSG_UPDATE: > + case LOGICAL_REP_MSG_DELETE: > + Assert(extra_data != NULL); > + > + /* > + * Compute size based on ApplyExecutionData. > + * The size of LogicalRepRelMapEntry can be > skipped because > + * it is obtained from hash_search in > logicalrep_rel_open. > + */ > + size += sizeof(ApplyExecutionData) + sizeof(EState) > + > + sizeof(ResultRelInfo) + > + sizeof(ResultRelInfo); > + > + /* > + * Add some extra size if the target relation > is partitioned. > + * PartitionTupleRouting isn't exported. > Therefore, call the > + * function that returns its size instead. > + */ > + if > (extra_data->relmapentry->localrel->rd_rel->relkind == > RELKIND_PARTITIONED_TABLE) > + size += sizeof(ModifyTableState) + > PartitionTupleRoutingSize(); > + break; Thanks a lot ! Fixed. > 2) Can this be part of PgStat_StatDBEntry, similar to tables, functions and > subworkers. It might be more appropriate to have it there instead of having > another global variable. > + * Stats of prepared transactions should be displayed > + * at either commit prepared or rollback prepared time, even when it's > + * after the server restart. We have the apply worker send those > +statistics > + * to the stats collector at prepare time and the startup process > +restore > + * those at restart if necessary. > + */ > +static HTAB *subWorkerPreparedXactSizeHash = NULL; > + > +/* Fixed. Also, its name was too long when aligned with other PgStat_StatDBEntry memebers. Thus I renamed it as subworkers_preparedsizes. This depends on v21 in [1] [1] - https://www.postgresql.org/message-id/CAD21AoAkd4YSoQUUFfpcrYOtkPRbninaw3sD0qc77nLW6Q89gg%40mail.gma... Best Regards, Takamichi Osumi Attachments: [application/octet-stream] v10-0001-Rename-existing-columns-of-pg_stat_subscription_.patch (15.4K, ../../TYCPR01MB8373FEB287F733C81C1E4D42ED989@TYCPR01MB8373.jpnprd01.prod.outlook.com/2-v10-0001-Rename-existing-columns-of-pg_stat_subscription_.patch) download | inline diff: From 6b329308d81ffc7140d51c2709f59d1dd57f31e5 Mon Sep 17 00:00:00 2001 From: Osumi Takamichi <[email protected]> Date: Mon, 15 Nov 2021 08:45:55 +0000 Subject: [PATCH v10 1/2] Rename existing columns of pg_stat_subscription_workers and export PartitionTupleRouting --- doc/src/sgml/monitoring.sgml | 24 +++++----- src/backend/catalog/system_views.sql | 10 ++--- src/backend/executor/execPartition.c | 70 ----------------------------- src/backend/utils/adt/pgstatfuncs.c | 20 ++++----- src/include/catalog/pg_proc.dat | 2 +- src/include/executor/execPartition.h | 69 ++++++++++++++++++++++++++++ src/test/regress/expected/rules.out | 12 ++--- src/test/subscription/t/026_error_report.pl | 8 ++-- 8 files changed, 107 insertions(+), 108 deletions(-) diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml index 4ee97db..8b7920c 100644 --- a/doc/src/sgml/monitoring.sgml +++ b/doc/src/sgml/monitoring.sgml @@ -3103,31 +3103,31 @@ SELECT pid, wait_event_type, wait_event FROM pg_stat_activity WHERE wait_event i <row> <entry role="catalog_table_entry"><para role="column_definition"> - <structfield>relid</structfield> <type>oid</type> + <structfield>last_error_relid</structfield> <type>oid</type> </para> <para> OID of the relation that the worker was processing when the - error occurred + last error occurred </para></entry> </row> <row> <entry role="catalog_table_entry"><para role="column_definition"> - <structfield>command</structfield> <type>text</type> + <structfield>last_error_command</structfield> <type>text</type> </para> <para> - Name of command being applied when the error occurred. This field - is always NULL if the error was reported during the initial data - copy. + Name of the last command being applied when the error occurred. + This field is always NULL if the error was reported during the + initial data copy. </para></entry> </row> <row> <entry role="catalog_table_entry"><para role="column_definition"> - <structfield>xid</structfield> <type>xid</type> + <structfield>last_error_xid</structfield> <type>xid</type> </para> <para> - Transaction ID of the publisher node being applied when the error + Transaction ID of the publisher node being applied when the last error occurred. This field is always NULL if the error was reported during the initial data copy. </para></entry> @@ -3135,19 +3135,19 @@ SELECT pid, wait_event_type, wait_event FROM pg_stat_activity WHERE wait_event i <row> <entry role="catalog_table_entry"><para role="column_definition"> - <structfield>error_count</structfield> <type>uint8</type> + <structfield>last_error_count</structfield> <type>uint8</type> </para> <para> - Number of consecutive times the error occurred + Number of consecutive times the last error occurred </para></entry> </row> <row> <entry role="catalog_table_entry"><para role="column_definition"> - <structfield>error_message</structfield> <type>text</type> + <structfield>last_error_message</structfield> <type>text</type> </para> <para> - The error message + The last error message </para></entry> </row> diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index cb2f77c..ecf1a0b 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1267,11 +1267,11 @@ CREATE VIEW pg_stat_subscription_workers AS w.subid, s.subname, w.subrelid, - w.relid, - w.command, - w.xid, - w.error_count, - w.error_message, + w.last_error_relid, + w.last_error_command, + w.last_error_xid, + w.last_error_count, + w.last_error_message, w.first_error_time, w.last_error_time FROM (SELECT diff --git a/src/backend/executor/execPartition.c b/src/backend/executor/execPartition.c index 5c723bc..2467f4c 100644 --- a/src/backend/executor/execPartition.c +++ b/src/backend/executor/execPartition.c @@ -34,76 +34,6 @@ #include "utils/rls.h" #include "utils/ruleutils.h" - -/*----------------------- - * PartitionTupleRouting - Encapsulates all information required to - * route a tuple inserted into a partitioned table to one of its leaf - * partitions. - * - * partition_root - * The partitioned table that's the target of the command. - * - * partition_dispatch_info - * Array of 'max_dispatch' elements containing a pointer to a - * PartitionDispatch object for every partitioned table touched by tuple - * routing. The entry for the target partitioned table is *always* - * present in the 0th element of this array. See comment for - * PartitionDispatchData->indexes for details on how this array is - * indexed. - * - * nonleaf_partitions - * Array of 'max_dispatch' elements containing pointers to fake - * ResultRelInfo objects for nonleaf partitions, useful for checking - * the partition constraint. - * - * num_dispatch - * The current number of items stored in the 'partition_dispatch_info' - * array. Also serves as the index of the next free array element for - * new PartitionDispatch objects that need to be stored. - * - * max_dispatch - * The current allocated size of the 'partition_dispatch_info' array. - * - * partitions - * Array of 'max_partitions' elements containing a pointer to a - * ResultRelInfo for every leaf partition touched by tuple routing. - * Some of these are pointers to ResultRelInfos which are borrowed out of - * the owning ModifyTableState node. The remainder have been built - * especially for tuple routing. See comment for - * PartitionDispatchData->indexes for details on how this array is - * indexed. - * - * is_borrowed_rel - * Array of 'max_partitions' booleans recording whether a given entry - * in 'partitions' is a ResultRelInfo pointer borrowed from the owning - * ModifyTableState node, rather than being built here. - * - * num_partitions - * The current number of items stored in the 'partitions' array. Also - * serves as the index of the next free array element for new - * ResultRelInfo objects that need to be stored. - * - * max_partitions - * The current allocated size of the 'partitions' array. - * - * memcxt - * Memory context used to allocate subsidiary structs. - *----------------------- - */ -struct PartitionTupleRouting -{ - Relation partition_root; - PartitionDispatch *partition_dispatch_info; - ResultRelInfo **nonleaf_partitions; - int num_dispatch; - int max_dispatch; - ResultRelInfo **partitions; - bool *is_borrowed_rel; - int num_partitions; - int max_partitions; - MemoryContext memcxt; -}; - /*----------------------- * PartitionDispatch - information about one partitioned table in a partition * hierarchy required to route a tuple to any of its partitions. A diff --git a/src/backend/utils/adt/pgstatfuncs.c b/src/backend/utils/adt/pgstatfuncs.c index 4e1bbcc..e7ff4ef 100644 --- a/src/backend/utils/adt/pgstatfuncs.c +++ b/src/backend/utils/adt/pgstatfuncs.c @@ -2441,15 +2441,15 @@ pg_stat_get_subscription_worker(PG_FUNCTION_ARGS) OIDOID, -1, 0); TupleDescInitEntry(tupdesc, (AttrNumber) 2, "subrelid", OIDOID, -1, 0); - TupleDescInitEntry(tupdesc, (AttrNumber) 3, "relid", + TupleDescInitEntry(tupdesc, (AttrNumber) 3, "last_error_relid", OIDOID, -1, 0); - TupleDescInitEntry(tupdesc, (AttrNumber) 4, "command", + TupleDescInitEntry(tupdesc, (AttrNumber) 4, "last_error_command", TEXTOID, -1, 0); - TupleDescInitEntry(tupdesc, (AttrNumber) 5, "xid", + TupleDescInitEntry(tupdesc, (AttrNumber) 5, "last_error_xid", XIDOID, -1, 0); - TupleDescInitEntry(tupdesc, (AttrNumber) 6, "error_count", + TupleDescInitEntry(tupdesc, (AttrNumber) 6, "last_error_count", INT8OID, -1, 0); - TupleDescInitEntry(tupdesc, (AttrNumber) 7, "error_message", + TupleDescInitEntry(tupdesc, (AttrNumber) 7, "last_error_message", TEXTOID, -1, 0); TupleDescInitEntry(tupdesc, (AttrNumber) 8, "first_error_time", TIMESTAMPTZOID, -1, 0); @@ -2471,28 +2471,28 @@ pg_stat_get_subscription_worker(PG_FUNCTION_ARGS) else nulls[i++] = true; - /* relid */ + /* last_error_relid */ if (OidIsValid(wentry->relid)) values[i++] = ObjectIdGetDatum(wentry->relid); else nulls[i++] = true; - /* command */ + /* last_error_command */ if (wentry->command != 0) values[i++] = CStringGetTextDatum(logicalrep_message_type(wentry->command)); else nulls[i++] = true; - /* xid */ + /* last_error_xid */ if (TransactionIdIsValid(wentry->xid)) values[i++] = TransactionIdGetDatum(wentry->xid); else nulls[i++] = true; - /* error_count */ + /* last_error_count */ values[i++] = Int64GetDatum(wentry->error_count); - /* error_message */ + /* last_error_message */ values[i++] = CStringGetTextDatum(wentry->error_message); /* first_error_time */ diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index 50e1c7b..25f685f 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -5391,7 +5391,7 @@ prorettype => 'record', proargtypes => 'oid oid', proallargtypes => '{oid,oid,oid,oid,oid,text,xid,int8,text,timestamptz,timestamptz}', proargmodes => '{i,i,o,o,o,o,o,o,o,o,o}', - proargnames => '{subid,subrelid,subid,subrelid,relid,command,xid,error_count,error_message,first_error_time,last_error_time}', + proargnames => '{subid,subrelid,subid,subrelid,last_error_relid,last_error_command,last_error_xid,last_error_count,last_error_message,first_error_time,last_error_time}', prosrc => 'pg_stat_get_subscription_worker' }, { oid => '6118', descr => 'statistics: information about subscription', proname => 'pg_stat_get_subscription', prorows => '10', proisstrict => 'f', diff --git a/src/include/executor/execPartition.h b/src/include/executor/execPartition.h index 694e38b..4c796ab 100644 --- a/src/include/executor/execPartition.h +++ b/src/include/executor/execPartition.h @@ -110,6 +110,75 @@ typedef struct PartitionPruneState PartitionPruningData *partprunedata[FLEXIBLE_ARRAY_MEMBER]; } PartitionPruneState; +/*----------------------- + * PartitionTupleRouting - Encapsulates all information required to + * route a tuple inserted into a partitioned table to one of its leaf + * partitions. + * + * partition_root + * The partitioned table that's the target of the command. + * + * partition_dispatch_info + * Array of 'max_dispatch' elements containing a pointer to a + * PartitionDispatch object for every partitioned table touched by tuple + * routing. The entry for the target partitioned table is *always* + * present in the 0th element of this array. See comment for + * PartitionDispatchData->indexes for details on how this array is + * indexed. + * + * nonleaf_partitions + * Array of 'max_dispatch' elements containing pointers to fake + * ResultRelInfo objects for nonleaf partitions, useful for checking + * the partition constraint. + * + * num_dispatch + * The current number of items stored in the 'partition_dispatch_info' + * array. Also serves as the index of the next free array element for + * new PartitionDispatch objects that need to be stored. + * + * max_dispatch + * The current allocated size of the 'partition_dispatch_info' array. + * + * partitions + * Array of 'max_partitions' elements containing a pointer to a + * ResultRelInfo for every leaf partition touched by tuple routing. + * Some of these are pointers to ResultRelInfos which are borrowed out of + * the owning ModifyTableState node. The remainder have been built + * especially for tuple routing. See comment for + * PartitionDispatchData->indexes for details on how this array is + * indexed. + * + * is_borrowed_rel + * Array of 'max_partitions' booleans recording whether a given entry + * in 'partitions' is a ResultRelInfo pointer borrowed from the owning + * ModifyTableState node, rather than being built here. + * + * num_partitions + * The current number of items stored in the 'partitions' array. Also + * serves as the index of the next free array element for new + * ResultRelInfo objects that need to be stored. + * + * max_partitions + * The current allocated size of the 'partitions' array. + * + * memcxt + * Memory context used to allocate subsidiary structs. + *----------------------- + */ +struct PartitionTupleRouting +{ + Relation partition_root; + PartitionDispatch *partition_dispatch_info; + ResultRelInfo **nonleaf_partitions; + int num_dispatch; + int max_dispatch; + ResultRelInfo **partitions; + bool *is_borrowed_rel; + int num_partitions; + int max_partitions; + MemoryContext memcxt; +}; + extern PartitionTupleRouting *ExecSetupPartitionTupleRouting(EState *estate, Relation rel); extern ResultRelInfo *ExecFindPartition(ModifyTableState *mtstate, diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out index cb6da2c..d60c5a5 100644 --- a/src/test/regress/expected/rules.out +++ b/src/test/regress/expected/rules.out @@ -2097,11 +2097,11 @@ pg_stat_subscription| SELECT su.oid AS subid, pg_stat_subscription_workers| SELECT w.subid, s.subname, w.subrelid, - w.relid, - w.command, - w.xid, - w.error_count, - w.error_message, + w.last_error_relid, + w.last_error_command, + w.last_error_xid, + w.last_error_count, + w.last_error_message, w.first_error_time, w.last_error_time FROM ( SELECT pg_subscription.oid AS subid, @@ -2112,7 +2112,7 @@ pg_stat_subscription_workers| SELECT w.subid, pg_subscription_rel.srrelid AS relid FROM pg_subscription_rel WHERE (pg_subscription_rel.srsubstate <> 'r'::"char")) sr, - (LATERAL pg_stat_get_subscription_worker(sr.subid, sr.relid) w(subid, subrelid, relid, command, xid, error_count, error_message, first_error_time, last_error_time) + (LATERAL pg_stat_get_subscription_worker(sr.subid, sr.relid) w(subid, subrelid, last_error_relid, last_error_command, last_error_xid, last_error_count, last_error_message, first_error_time, last_error_time) JOIN pg_subscription s ON ((w.subid = s.oid))); pg_stat_sys_indexes| SELECT pg_stat_all_indexes.relid, pg_stat_all_indexes.indexrelid, diff --git a/src/test/subscription/t/026_error_report.pl b/src/test/subscription/t/026_error_report.pl index 1227654..ca30ab2 100644 --- a/src/test/subscription/t/026_error_report.pl +++ b/src/test/subscription/t/026_error_report.pl @@ -15,8 +15,8 @@ sub test_subscription_error my $check_sql = qq[ SELECT count(1) > 0 FROM pg_stat_subscription_workers -WHERE relid = '$relname'::regclass]; - $check_sql .= " AND xid = '$xid'::xid;" if $xid ne ''; +WHERE last_error_relid = '$relname'::regclass]; + $check_sql .= " AND last_error_xid = '$xid'::xid;" if $xid ne ''; # Wait for the error statistics to be updated. $node->poll_query_until( @@ -26,9 +26,9 @@ WHERE relid = '$relname'::regclass]; my $result = $node->safe_psql( 'postgres', qq[ -SELECT subname, command, relid::regclass, error_count > 0 +SELECT subname, last_error_command, last_error_relid::regclass, last_error_count > 0 FROM pg_stat_subscription_workers -WHERE relid = '$relname'::regclass; +WHERE last_error_relid = '$relname'::regclass; ]); is($result, $expected_error, $msg); } -- 2.2.0 [application/octet-stream] v10-0002-Extend-pg_stat_subscription_workers-to-include-g.patch (53.2K, ../../TYCPR01MB8373FEB287F733C81C1E4D42ED989@TYCPR01MB8373.jpnprd01.prod.outlook.com/3-v10-0002-Extend-pg_stat_subscription_workers-to-include-g.patch) download | inline diff: From fe2554ec7f667cc28a4125aa48fb74d904e52da0 Mon Sep 17 00:00:00 2001 From: Osumi Takamichi <[email protected]> Date: Mon, 15 Nov 2021 08:47:04 +0000 Subject: [PATCH v10 2/2] Extend pg_stat_subscription_workers to include general transaction statistics Categorize transactions of logical replication subscriber into three types (commit, error, abort) and introduce cumulative columns of those numbers and amounts of consumed data during message apply respectively. The calculation of consumed resources by subscriber's apply is computed based on the data structure for message apply and extra data, which is different from that of publisher's decoding processing. At present, there's no special consideration to spool file statistics such as amount of data spooled to disk or its count. But, this can be added later. The amount of data used by apply for STREAM COMMIT and STREAM PREPARE is transaction data consumption and can be regarded as data that should be added to column of this commit. This size of prepared transaction becomes persistent to conclude the appropriate category for the transaction at either commit prepared or rollback prepared time. Author: Takamichi Osumi Discussed & Reviewed-by: Amit Kapila, Masahiko Sawada, Hou Zhijie, Greg Nancarrow, Vignesh C, Ajin Cherian Discussion: https://www.postgresql.org/message-id/OSBPR01MB48887CA8F40C8D984A6DC00CED199%40OSBPR01MB4888.jpnprd01.prod.outlook.com --- doc/src/sgml/monitoring.sgml | 71 ++++- src/backend/catalog/system_views.sql | 6 + src/backend/postmaster/pgstat.c | 324 ++++++++++++++++++++++- src/backend/replication/logical/proto.c | 12 + src/backend/replication/logical/tablesync.c | 6 + src/backend/replication/logical/worker.c | 169 +++++++++++- src/backend/utils/adt/pgstatfuncs.c | 36 ++- src/include/catalog/pg_proc.dat | 6 +- src/include/pgstat.h | 105 +++++++- src/include/replication/logicalworker.h | 5 + src/test/regress/expected/rules.out | 8 +- src/test/subscription/t/027_worker_xact_stats.pl | 146 ++++++++++ src/tools/pgindent/typedefs.list | 4 + 13 files changed, 875 insertions(+), 23 deletions(-) create mode 100644 src/test/subscription/t/027_worker_xact_stats.pl diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml index 8b7920c..f0b9f35 100644 --- a/doc/src/sgml/monitoring.sgml +++ b/doc/src/sgml/monitoring.sgml @@ -629,8 +629,8 @@ postgres 27093 0.0 0.0 30096 2752 ? Ss 11:34 0:00 postgres: ser <row> <entry><structname>pg_stat_subscription_workers</structname><indexterm><primary>pg_stat_subscription_workers</primary></indexterm></entry> - <entry>One row per subscription worker, showing statistics about errors - that occurred on that subscription worker. + <entry>One row per subscription worker, showing transaction statistics + and information about errors that occurred on that subscription worker. See <link linkend="monitoring-pg-stat-subscription-workers"> <structname>pg_stat_subscription_workers</structname></link> for details. </entry> @@ -3052,10 +3052,12 @@ SELECT pid, wait_event_type, wait_event FROM pg_stat_activity WHERE wait_event i <para> The <structname>pg_stat_subscription_workers</structname> view will contain - one row per subscription error reported by workers applying logical + one row per subscription, showing corresponding transaction statistics and + information about the error reported by workers applying logical replication changes and workers handling the initial data copy of the subscribed tables. The statistics entry is removed when the subscription - the worker is running on is removed. + the worker is running on is removed. The statistics of transaction size is + utilized only by the apply worker. </para> <table id="pg-stat-subscription-workers" xreflabel="pg_stat_subscription_workers"> @@ -3103,6 +3105,67 @@ SELECT pid, wait_event_type, wait_event FROM pg_stat_activity WHERE wait_event i <row> <entry role="catalog_table_entry"><para role="column_definition"> + <structfield>commit_count</structfield> <type>bigint</type> + </para> + <para> + Number of transactions successfully applied in this subscription. + COMMIT, COMMIT of streaming transaction and COMMIT PREPARED increments + this counter. + </para></entry> + </row> + + <row> + <entry role="catalog_table_entry"><para role="column_definition"> + <structfield>commit_bytes</structfield> <type>bigint</type> + </para> + <para> + Amount of data (in bytes) successfully applied in this subscription, + across <literal>commit_count</literal> transactions. + </para></entry> + </row> + + <row> + <entry role="catalog_table_entry"><para role="column_definition"> + <structfield>error_count</structfield> <type>bigint</type> + </para> + <para> + Number of transactions that failed to be applied by the table + sync worker or main apply worker in this subscription. + </para></entry> + </row> + + <row> + <entry role="catalog_table_entry"><para role="column_definition"> + <structfield>error_bytes</structfield> <type>bigint</type> + </para> + <para> + Amount of data (in bytes) unsuccessfully applied in this subscription, + across <literal>error_count</literal> transactions. + </para></entry> + </row> + + <row> + <entry role="catalog_table_entry"><para role="column_definition"> + <structfield>abort_count</structfield> <type>bigint</type> + </para> + <para> + Number of transactions aborted in this subscription. + ROLLBACK PREPARED increments this counter. + </para></entry> + </row> + + <row> + <entry role="catalog_table_entry"><para role="column_definition"> + <structfield>abort_bytes</structfield> <type>bigint</type> + </para> + <para> + Amount of data (in bytes) aborted in this subscription, + across <literal>abort_count</literal> transactions. + </para></entry> + </row> + + <row> + <entry role="catalog_table_entry"><para role="column_definition"> <structfield>last_error_relid</structfield> <type>oid</type> </para> <para> diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index ecf1a0b..a8880e8 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1267,6 +1267,12 @@ CREATE VIEW pg_stat_subscription_workers AS w.subid, s.subname, w.subrelid, + w.commit_count, + w.commit_bytes, + w.error_count, + w.error_bytes, + w.abort_count, + w.abort_bytes, w.last_error_relid, w.last_error_command, w.last_error_xid, diff --git a/src/backend/postmaster/pgstat.c b/src/backend/postmaster/pgstat.c index a620379..c4df010 100644 --- a/src/backend/postmaster/pgstat.c +++ b/src/backend/postmaster/pgstat.c @@ -53,6 +53,7 @@ #include "postmaster/fork_process.h" #include "postmaster/interrupt.h" #include "postmaster/postmaster.h" +#include "replication/logicalworker.h" #include "replication/slot.h" #include "replication/walsender.h" #include "storage/backendid.h" @@ -325,11 +326,14 @@ static PgStat_StatTabEntry *pgstat_get_tab_entry(PgStat_StatDBEntry *dbentry, static PgStat_StatSubWorkerEntry *pgstat_get_subworker_entry(PgStat_StatDBEntry *dbentry, Oid subid, Oid subrelid, bool create); +static PgStat_SW_PreparedXactEntry *pgstat_get_subworker_prepared_txn(Oid databaseid, Oid subid, + char *gid, bool create); + static void pgstat_write_statsfiles(bool permanent, bool allDbs); static void pgstat_write_db_statsfile(PgStat_StatDBEntry *dbentry, bool permanent); static HTAB *pgstat_read_statsfiles(Oid onlydb, bool permanent, bool deep); static void pgstat_read_db_statsfile(Oid databaseid, HTAB *tabhash, HTAB *funchash, - HTAB *subworkerhash, bool permanent); + HTAB *subworkerhash, HTAB *preparedtxnhash, bool permanent); static void backend_read_statsfile(void); static bool pgstat_write_statsfile_needed(void); @@ -382,6 +386,9 @@ static void pgstat_recv_replslot(PgStat_MsgReplSlot *msg, int len); static void pgstat_recv_tempfile(PgStat_MsgTempFile *msg, int len); static void pgstat_recv_subscription_purge(PgStat_MsgSubscriptionPurge *msg, int len); static void pgstat_recv_subworker_error(PgStat_MsgSubWorkerError *msg, int len); +static void pgstat_recv_subworker_xact_end(PgStat_MsgSubWorkerXactEnd *msg, int len); +static void pgstat_recv_subworker_twophase_xact(PgStat_MsgSubWorkerTwophaseXact *msg, int len); + /* ------------------------------------------------------------ * Public functions called from postmaster follow @@ -1930,6 +1937,61 @@ pgstat_report_replslot_drop(const char *slotname) } /* ---------- + * pgstat_report_subworker_xact_end() - + * + * Tell the collector that worker transaction has successfully completed. + * ---------- + */ +void +pgstat_report_subworker_xact_end(Oid subid, Oid subrel, + LogicalRepMsgType command, PgStat_Counter xact_size) +{ + PgStat_MsgSubWorkerXactEnd msg; + + Assert(command == 0 /* table sync worker */ || + command == LOGICAL_REP_MSG_COMMIT || + command == LOGICAL_REP_MSG_STREAM_COMMIT); + + pgstat_setheader(&msg.m_hdr, PGSTAT_MTYPE_SUBWORKERXACTEND); + msg.m_databaseid = MyDatabaseId; + msg.m_subid = subid; + msg.m_subrelid = subrel; + msg.m_command = command; + msg.m_xact_bytes = xact_size; + pgstat_send(&msg, sizeof(PgStat_MsgSubWorkerXactEnd)); + + reset_apply_error_context_xact_size(); +} + +/* ---------- + * pgstat_report_subworker_twophase_xact() - + * + * Tell the collector that worker transaction has done 2PC related operation. + * ---------- + */ +void +pgstat_report_subworker_twophase_xact(Oid subid, LogicalRepMsgType command, + PgStat_Counter xact_size, char *gid) +{ + PgStat_MsgSubWorkerTwophaseXact msg; + + Assert(command == LOGICAL_REP_MSG_PREPARE || + command == LOGICAL_REP_MSG_STREAM_PREPARE || + command == LOGICAL_REP_MSG_COMMIT_PREPARED || + command == LOGICAL_REP_MSG_ROLLBACK_PREPARED); + + /* setup the message */ + pgstat_setheader(&msg.m_hdr, PGSTAT_MTYPE_SUBWORKERTWOPHASEXACT); + msg.m_databaseid = MyDatabaseId; + msg.m_subid = subid; + msg.m_command = command; + strlcpy(msg.m_gid, gid, sizeof(msg.m_gid)); + msg.m_xact_bytes = xact_size; + pgstat_send(&msg, sizeof(PgStat_MsgSubWorkerTwophaseXact)); + reset_apply_error_context_xact_size(); +} + +/* ---------- * pgstat_report_subworker_error() - * * Tell the collector about the subscription worker error. @@ -1950,6 +2012,7 @@ pgstat_report_subworker_error(Oid subid, Oid subrelid, Oid relid, msg.m_databaseid = MyDatabaseId; msg.m_subid = subid; msg.m_subrelid = subrelid; + msg.m_xact_error_bytes = get_apply_error_context_xact_size(); msg.m_relid = relid; msg.m_command = command; msg.m_xid = xid; @@ -1957,6 +2020,8 @@ pgstat_report_subworker_error(Oid subid, Oid subrelid, Oid relid, strlcpy(msg.m_message, errmsg, PGSTAT_SUBWORKERERROR_MSGLEN); pgstat_send(&msg, len); + + reset_apply_error_context_xact_size(); } /* ---------- @@ -3727,6 +3792,14 @@ PgstatCollectorMain(int argc, char *argv[]) pgstat_recv_subworker_error(&msg.msg_subworkererror, len); break; + case PGSTAT_MTYPE_SUBWORKERXACTEND: + pgstat_recv_subworker_xact_end(&msg.msg_subworkerxactend, len); + break; + + case PGSTAT_MTYPE_SUBWORKERTWOPHASEXACT: + pgstat_recv_subworker_twophase_xact(&msg.msg_subworkertwophasexact, len); + break; + default: break; } @@ -3833,6 +3906,14 @@ reset_dbentry_counters(PgStat_StatDBEntry *dbentry) PGSTAT_SUBWORKER_HASH_SIZE, &hash_ctl, HASH_ELEM | HASH_BLOBS); + + hash_ctl.keysize = sizeof(PgStat_SW_PreparedXactKey); + hash_ctl.entrysize = sizeof(PgStat_SW_PreparedXactEntry); + hash_ctl.hcxt = pgStatLocalContext; + dbentry->subworkers_preparedsizes = hash_create("Subscription worker stats of prepared txn", + PGSTAT_SUBWORKER_HASH_SIZE, + &hash_ctl, + HASH_ELEM | HASH_STRINGS | HASH_CONTEXT); } /* @@ -3943,6 +4024,12 @@ pgstat_get_subworker_entry(PgStat_StatDBEntry *dbentry, Oid subid, Oid subrelid, /* If not found, initialize the new one */ if (create && !found) { + subwentry->xact_commit_count = 0; + subwentry->xact_commit_bytes = 0; + subwentry->xact_error_count = 0; + subwentry->xact_error_bytes = 0; + subwentry->xact_abort_count = 0; + subwentry->xact_abort_bytes = 0; subwentry->relid = InvalidOid; subwentry->command = 0; subwentry->xid = InvalidTransactionId; @@ -4154,9 +4241,11 @@ pgstat_write_db_statsfile(PgStat_StatDBEntry *dbentry, bool permanent) HASH_SEQ_STATUS tstat; HASH_SEQ_STATUS fstat; HASH_SEQ_STATUS sstat; + HASH_SEQ_STATUS pstat; PgStat_StatTabEntry *tabentry; PgStat_StatFuncEntry *funcentry; PgStat_StatSubWorkerEntry *subwentry; + PgStat_SW_PreparedXactEntry *pentry; FILE *fpout; int32 format_id; Oid dbid = dbentry->databaseid; @@ -4223,6 +4312,17 @@ pgstat_write_db_statsfile(PgStat_StatDBEntry *dbentry, bool permanent) } /* + * Write subscription worker's prepared transaction struct + */ + hash_seq_init(&pstat, dbentry->subworkers_preparedsizes); + while ((pentry = (PgStat_SW_PreparedXactEntry *) hash_seq_search(&pstat)) != NULL) + { + fputc('P', fpout); + rc = fwrite(pentry, sizeof(PgStat_SW_PreparedXactEntry), 1, fpout); + (void) rc; /* we'll check for error with ferror */ + } + + /* * No more output to be done. Close the temp file and replace the old * pgstat.stat with it. The ferror() check replaces testing for error * after each individual fputc or fwrite above. @@ -4461,6 +4561,7 @@ pgstat_read_statsfiles(Oid onlydb, bool permanent, bool deep) dbentry->tables = NULL; dbentry->functions = NULL; dbentry->subworkers = NULL; + dbentry->subworkers_preparedsizes = NULL; /* * In the collector, disregard the timestamp we read from the @@ -4506,6 +4607,14 @@ pgstat_read_statsfiles(Oid onlydb, bool permanent, bool deep) &hash_ctl, HASH_ELEM | HASH_BLOBS | HASH_CONTEXT); + hash_ctl.keysize = sizeof(PgStat_SW_PreparedXactKey); + hash_ctl.entrysize = sizeof(PgStat_SW_PreparedXactEntry); + hash_ctl.hcxt = pgStatLocalContext; + dbentry->subworkers_preparedsizes = hash_create("Subscription worker stats of prepared txn", + PGSTAT_SUBWORKER_HASH_SIZE, + &hash_ctl, + HASH_ELEM | HASH_STRINGS | HASH_CONTEXT); + /* * If requested, read the data from the database-specific * file. Otherwise we just leave the hashtables empty. @@ -4515,6 +4624,7 @@ pgstat_read_statsfiles(Oid onlydb, bool permanent, bool deep) dbentry->tables, dbentry->functions, dbentry->subworkers, + dbentry->subworkers_preparedsizes, permanent); break; @@ -4599,7 +4709,7 @@ done: */ static void pgstat_read_db_statsfile(Oid databaseid, HTAB *tabhash, HTAB *funchash, - HTAB *subworkerhash, bool permanent) + HTAB *subworkerhash, HTAB *preparedtxnhash, bool permanent) { PgStat_StatTabEntry *tabentry; PgStat_StatTabEntry tabbuf; @@ -4755,6 +4865,32 @@ pgstat_read_db_statsfile(Oid databaseid, HTAB *tabhash, HTAB *funchash, memcpy(subwentry, &subwbuf, sizeof(subwbuf)); break; + case 'P': + { + PgStat_SW_PreparedXactEntry buff; + PgStat_SW_PreparedXactEntry *prepared_xact_entry; + + if (fread(&buff, 1, sizeof(PgStat_SW_PreparedXactEntry), + fpin) != sizeof(PgStat_SW_PreparedXactEntry)) + { + ereport(pgStatRunningInCollector ? LOG : WARNING, + (errmsg("corrupted statistics file \"%s\"", + statfile))); + goto done; + } + + if (preparedtxnhash == NULL) + break; + + prepared_xact_entry = + (PgStat_SW_PreparedXactEntry *) hash_search(preparedtxnhash, + (void *) &buff.key, + HASH_ENTER, NULL); + + memcpy(prepared_xact_entry, &buff, sizeof(PgStat_SW_PreparedXactEntry)); + break; + } + /* * 'E' The EOF marker of a complete stats file. */ @@ -5430,6 +5566,8 @@ pgstat_recv_dropdb(PgStat_MsgDropdb *msg, int len) hash_destroy(dbentry->functions); if (dbentry->subworkers != NULL) hash_destroy(dbentry->subworkers); + if (dbentry->subworkers_preparedsizes != NULL) + hash_destroy(dbentry->subworkers_preparedsizes); if (hash_search(pgStatDBHash, (void *) &dbid, @@ -5469,10 +5607,13 @@ pgstat_recv_resetcounter(PgStat_MsgResetcounter *msg, int len) hash_destroy(dbentry->functions); if (dbentry->subworkers != NULL) hash_destroy(dbentry->subworkers); + if (dbentry->subworkers_preparedsizes != NULL) + hash_destroy(dbentry->subworkers_preparedsizes); dbentry->tables = NULL; dbentry->functions = NULL; dbentry->subworkers = NULL; + dbentry->subworkers_preparedsizes = NULL; /* * Reset database-level stats, too. This creates empty hash tables for @@ -5548,10 +5689,28 @@ pgstat_recv_resetsinglecounter(PgStat_MsgResetsinglecounter *msg, int len) else if (msg->m_resettype == RESET_SUBWORKER) { PgStat_StatSubWorkerKey key; + PgStat_SW_PreparedXactKey pkey; key.subid = msg->m_objectid; key.subrelid = msg->m_subobjectid; (void) hash_search(dbentry->subworkers, (void *) &key, HASH_REMOVE, NULL); + + /* + * Clean up entry of prepared size hash as well. + * + * There's no gid input here as a part of key for + * PgStat_SW_PreparedXactEntry. Also any valid 'relid' indicates the + * entry is created by table sync but it has nothing to do with the + * two phase operation of apply. Proceed with this removal only when + * the specified subrelid isn't invalid. + */ + if (msg->m_subobjectid == InvalidOid) + { + pkey.subid = msg->m_objectid; + (void) hash_search(dbentry->subworkers_preparedsizes, + (void *) &pkey, HASH_REMOVE, NULL); + } + } } @@ -6107,6 +6266,7 @@ pgstat_recv_subscription_purge(PgStat_MsgSubscriptionPurge *msg, int len) HASH_SEQ_STATUS hstat; PgStat_StatDBEntry *dbentry; PgStat_StatSubWorkerEntry *subwentry; + PgStat_SW_PreparedXactEntry *prepared_txn_entry; dbentry = pgstat_get_db_entry(msg->m_databaseid, false); @@ -6128,6 +6288,127 @@ pgstat_recv_subscription_purge(PgStat_MsgSubscriptionPurge *msg, int len) } } } + + /* Remove associated prepared transaction stats */ + if (!dbentry->subworkers_preparedsizes) + { + hash_seq_init(&hstat, dbentry->subworkers_preparedsizes); + while ((prepared_txn_entry = (PgStat_SW_PreparedXactEntry *) hash_seq_search(&hstat)) != NULL) + { + for (int i = 0; i < msg->m_nentries; i++) + { + if (prepared_txn_entry->key.subid == msg->m_subids[i]) + (void) hash_search(dbentry->subworkers_preparedsizes, + (void *) &(prepared_txn_entry->key), + HASH_REMOVE, NULL); + break; + } + } + } +} + +/* ---------- + * pgstat_recv_subworker_xact_end() - + * + * Process a SUBWORKERXACTEND message. + * ---------- + */ +static void +pgstat_recv_subworker_xact_end(PgStat_MsgSubWorkerXactEnd *msg, int len) +{ + PgStat_StatDBEntry *dbentry; + PgStat_StatSubWorkerEntry *wentry; + + dbentry = pgstat_get_db_entry(msg->m_databaseid, true); + wentry = pgstat_get_subworker_entry(dbentry, msg->m_subid, + msg->m_subrelid, true); + Assert(wentry); + + /* table sync worker */ + if (msg->m_command == 0) + wentry->xact_commit_count++; + else + { + /* apply worker */ + switch (msg->m_command) + { + case LOGICAL_REP_MSG_COMMIT: + case LOGICAL_REP_MSG_STREAM_COMMIT: + wentry->xact_commit_count++; + wentry->xact_commit_bytes += msg->m_xact_bytes; + break; + default: + elog(ERROR, "unexpected logical message type as normal apply end"); + break; + } + } +} + +/* ---------- + * pgstat_recv_subworker_twophase_xact() - + * + * Process a SUBWORKERTWOPHASEXACT message. + * ---------- + */ +static void +pgstat_recv_subworker_twophase_xact(PgStat_MsgSubWorkerTwophaseXact *msg, int len) +{ + PgStat_StatDBEntry *dbentry; + PgStat_SW_PreparedXactEntry *pentry; + PgStat_StatSubWorkerEntry *wentry; + PgStat_SW_PreparedXactKey key; + + pentry = pgstat_get_subworker_prepared_txn(msg->m_databaseid, + msg->m_subid, + msg->m_gid, true); + Assert(pentry); + switch (msg->m_command) + { + case LOGICAL_REP_MSG_PREPARE: + case LOGICAL_REP_MSG_STREAM_PREPARE: + + /* + * Make each size of prepared transaction persistent so that we + * can update stats over the server restart and make prepared + * stats updated when commit prepared or rollback prepared + * arrives. + */ + pentry->subid = msg->m_subid; + strlcpy(pentry->gid, msg->m_gid, sizeof(pentry->gid)); + pentry->xact_size = msg->m_xact_bytes; + break; + + case LOGICAL_REP_MSG_COMMIT_PREPARED: + case LOGICAL_REP_MSG_ROLLBACK_PREPARED: + /* Update exported xact stats now */ + dbentry = pgstat_get_db_entry(msg->m_databaseid, true); + wentry = pgstat_get_subworker_entry(dbentry, + msg->m_subid, + InvalidOid /* apply worker */ , + true); + Assert(wentry); + if (msg->m_command == LOGICAL_REP_MSG_COMMIT_PREPARED) + { + wentry->xact_commit_count++; + wentry->xact_commit_bytes += pentry->xact_size; + } + else + { + wentry->xact_abort_count++; + wentry->xact_abort_bytes += pentry->xact_size; + } + + /* Clean up this gid from transaction size hash */ + key.subid = pentry->subid; + strlcpy(key.gid, msg->m_gid, sizeof(key.gid)); + (void) hash_search(dbentry->subworkers_preparedsizes, + (void *) &key, HASH_REMOVE, NULL); + break; + + default: + elog(ERROR, "unexpected logical message type as prepare transaction"); + break; + } } /* ---------- @@ -6149,6 +6430,10 @@ pgstat_recv_subworker_error(PgStat_MsgSubWorkerError *msg, int len) msg->m_subrelid, true); Assert(subwentry); + /* general transaction stats for error */ + subwentry->xact_error_count++; + subwentry->xact_error_bytes += msg->m_xact_error_bytes; + /* * Update only the counter and last error timestamp if we received * the same error again @@ -6396,3 +6681,38 @@ pgstat_count_slru_truncate(int slru_idx) { slru_entry(slru_idx)->m_truncate += 1; } + + /* ---------- + * pgstat_get_subworker_prepared_txn + * + * Return subscription worker entry with the given subscription OID and + * gid. + * ---------- + */ +static PgStat_SW_PreparedXactEntry * +pgstat_get_subworker_prepared_txn(Oid databaseid, Oid subid, + char *gid, bool create) +{ + PgStat_StatDBEntry *dbentry; + PgStat_SW_PreparedXactKey key; + PgStat_SW_PreparedXactEntry *pentry; + HASHACTION action; + bool found; + + dbentry = pgstat_get_db_entry(databaseid, true); + key.subid = subid; + strlcpy(key.gid, gid, sizeof(key.gid)); + action = (create ? HASH_ENTER : HASH_FIND); + pentry = (PgStat_SW_PreparedXactEntry *) hash_search(dbentry->subworkers_preparedsizes, + (void *) &key, + action, &found); + + if (create && !found) + { + pentry->subid = 0; + pentry->gid[0] = '\0'; + pentry->xact_size = 0; + } + + return pentry; +} diff --git a/src/backend/replication/logical/proto.c b/src/backend/replication/logical/proto.c index 9f5bf4b..3cd933a 100644 --- a/src/backend/replication/logical/proto.c +++ b/src/backend/replication/logical/proto.c @@ -17,6 +17,7 @@ #include "catalog/pg_type.h" #include "libpq/pqformat.h" #include "replication/logicalproto.h" +#include "replication/logicalworker.h" #include "utils/lsyscache.h" #include "utils/syscache.h" @@ -842,6 +843,7 @@ logicalrep_read_tuple(StringInfo in, LogicalRepTupleData *tuple) { int i; int natts; + Size total_size; /* Get number of attributes */ natts = pq_getmsgint(in, 2); @@ -850,6 +852,7 @@ logicalrep_read_tuple(StringInfo in, LogicalRepTupleData *tuple) tuple->colvalues = (StringInfoData *) palloc0(natts * sizeof(StringInfoData)); tuple->colstatus = (char *) palloc(natts * sizeof(char)); tuple->ncols = natts; + total_size = natts * (sizeof(char) + sizeof(StringInfoData)); /* Read the data */ for (i = 0; i < natts; i++) @@ -880,6 +883,9 @@ logicalrep_read_tuple(StringInfo in, LogicalRepTupleData *tuple) value->len = len; value->cursor = 0; value->maxlen = len; + + /* memory for tuple */ + total_size += len + 1; break; case LOGICALREP_COLUMN_BINARY: len = pq_getmsgint(in, 4); /* read length */ @@ -893,11 +899,17 @@ logicalrep_read_tuple(StringInfo in, LogicalRepTupleData *tuple) value->len = len; value->cursor = 0; value->maxlen = len; + + /* memory for tuple */ + total_size += len + 1; break; default: elog(ERROR, "unrecognized data representation type '%c'", kind); } } + + /* Record memory consumption for tuple */ + add_apply_error_context_xact_size(total_size); } /* diff --git a/src/backend/replication/logical/tablesync.c b/src/backend/replication/logical/tablesync.c index f07983a..5bb1d63 100644 --- a/src/backend/replication/logical/tablesync.c +++ b/src/backend/replication/logical/tablesync.c @@ -1149,6 +1149,12 @@ copy_table_done: MyLogicalRepWorker->relstate_lsn = *origin_startpos; SpinLockRelease(&MyLogicalRepWorker->relmutex); + /* Report the success of table sync. */ + pgstat_report_subworker_xact_end(MyLogicalRepWorker->subid, + MyLogicalRepWorker->relid, + 0 /* no logical message type */ , + 0 /* xact size */ ); + /* * Finally, wait until the main apply worker tells us to catch up and then * return to let LogicalRepApplyLoop do it. diff --git a/src/backend/replication/logical/worker.c b/src/backend/replication/logical/worker.c index 2e79302..6617437 100644 --- a/src/backend/replication/logical/worker.c +++ b/src/backend/replication/logical/worker.c @@ -221,6 +221,18 @@ typedef struct ApplyErrorCallbackArg LogicalRepMsgType command; /* 0 if invalid */ LogicalRepRelMapEntry *rel; + /* + * Store data size of this transaction. + * + * The byte size of transaction on the publisher is calculated by + * ReorderBufferChangeSize() based on the ReorderBufferChange structure. + * But on the subscriber, consumed resources are not same as the + * publisher's decoding processsing and required to be computed in + * different way. Therefore, the exact same byte size is not restored on + * the subscriber usually. + */ + PgStat_Counter bytes; + /* Remote node information */ int remote_attnum; /* -1 if invalid */ TransactionId remote_xid; @@ -231,6 +243,7 @@ static ApplyErrorCallbackArg apply_error_callback_arg = { .command = 0, .rel = NULL, + .bytes = 0, .remote_attnum = -1, .remote_xid = InvalidTransactionId, .ts = 0, @@ -293,6 +306,7 @@ static inline void cleanup_subxact_info(void); static void stream_cleanup_files(Oid subid, TransactionId xid); static void stream_open_file(Oid subid, TransactionId xid, bool first); static void stream_write_change(char action, StringInfo s); + static void stream_close_file(void); static void send_feedback(XLogRecPtr recvpos, bool force, bool requestReply); @@ -304,6 +318,9 @@ static void maybe_reread_subscription(void); /* prototype needed because of stream_commit */ static void apply_dispatch(StringInfo s); +static void update_apply_change_size(LogicalRepMsgType action, + bool is_partitioned); + static void apply_handle_commit_internal(LogicalRepCommitData *commit_data); static void apply_handle_insert_internal(ApplyExecutionData *edata, ResultRelInfo *relinfo, @@ -818,6 +835,11 @@ apply_handle_commit(StringInfo s) /* Process any tables that are being synchronized in parallel. */ process_syncing_tables(commit_data.end_lsn); + pgstat_report_subworker_xact_end(MyLogicalRepWorker->subid, + InvalidOid, + LOGICAL_REP_MSG_COMMIT, + get_apply_error_context_xact_size()); + pgstat_report_activity(STATE_IDLE, NULL); reset_apply_error_context_info(); } @@ -922,6 +944,13 @@ apply_handle_prepare(StringInfo s) /* Process any tables that are being synchronized in parallel. */ process_syncing_tables(prepare_data.end_lsn); + /* Update the memory consumption of prepare and report */ + update_apply_change_size(LOGICAL_REP_MSG_PREPARE, false); + pgstat_report_subworker_twophase_xact(MyLogicalRepWorker->subid, + LOGICAL_REP_MSG_PREPARE, + get_apply_error_context_xact_size(), + prepare_data.gid); + pgstat_report_activity(STATE_IDLE, NULL); reset_apply_error_context_info(); } @@ -963,6 +992,13 @@ apply_handle_commit_prepared(StringInfo s) /* Process any tables that are being synchronized in parallel. */ process_syncing_tables(prepare_data.end_lsn); + /* Update the transaction size and report */ + update_apply_change_size(LOGICAL_REP_MSG_COMMIT_PREPARED, false); + pgstat_report_subworker_twophase_xact(MyLogicalRepWorker->subid, + LOGICAL_REP_MSG_COMMIT_PREPARED, + get_apply_error_context_xact_size(), + prepare_data.gid); + pgstat_report_activity(STATE_IDLE, NULL); reset_apply_error_context_info(); } @@ -1014,6 +1050,13 @@ apply_handle_rollback_prepared(StringInfo s) /* Process any tables that are being synchronized in parallel. */ process_syncing_tables(rollback_data.rollback_end_lsn); + /* Update the transaction size and report */ + update_apply_change_size(LOGICAL_REP_MSG_ROLLBACK_PREPARED, false); + pgstat_report_subworker_twophase_xact(MyLogicalRepWorker->subid, + LOGICAL_REP_MSG_ROLLBACK_PREPARED, + get_apply_error_context_xact_size(), + rollback_data.gid); + pgstat_report_activity(STATE_IDLE, NULL); reset_apply_error_context_info(); } @@ -1066,6 +1109,15 @@ apply_handle_stream_prepare(StringInfo s) /* Process any tables that are being synchronized in parallel. */ process_syncing_tables(prepare_data.end_lsn); + /* + * Report the prepared streaming xact size to the stats collector in a + * prepared xact manner to make it survive over the restart. + */ + pgstat_report_subworker_twophase_xact(MyLogicalRepWorker->subid, + LOGICAL_REP_MSG_STREAM_PREPARE, + get_apply_error_context_xact_size(), + prepare_data.gid); + pgstat_report_activity(STATE_IDLE, NULL); reset_apply_error_context_info(); @@ -1438,6 +1490,12 @@ apply_handle_stream_commit(StringInfo s) /* Process any tables that are being synchronized in parallel. */ process_syncing_tables(commit_data.end_lsn); + /* Report and clean up the xid */ + pgstat_report_subworker_xact_end(MyLogicalRepWorker->subid, + InvalidOid, + LOGICAL_REP_MSG_STREAM_COMMIT, + get_apply_error_context_xact_size()); + pgstat_report_activity(STATE_IDLE, NULL); reset_apply_error_context_info(); @@ -1579,6 +1637,10 @@ apply_handle_insert(StringInfo s) slot_fill_defaults(rel, estate, remoteslot); MemoryContextSwitchTo(oldctx); + /* Update transaction size */ + update_apply_change_size(LOGICAL_REP_MSG_INSERT, + rel->localrel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE); + /* For a partitioned table, insert the tuple into a partition. */ if (rel->localrel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE) apply_handle_tuple_routing(edata, @@ -1736,7 +1798,11 @@ apply_handle_update(StringInfo s) has_oldtup ? &oldtup : &newtup); MemoryContextSwitchTo(oldctx); - /* For a partitioned table, apply update to correct partition. */ + /* Update transaction size */ + update_apply_change_size(LOGICAL_REP_MSG_UPDATE, + rel->localrel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE); + + /* For a partitioned table, apply update to corect partition. */ if (rel->localrel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE) apply_handle_tuple_routing(edata, remoteslot, &newtup, CMD_UPDATE); @@ -1870,6 +1936,10 @@ apply_handle_delete(StringInfo s) slot_store_data(remoteslot, rel, &oldtup); MemoryContextSwitchTo(oldctx); + /* Update transaction size */ + update_apply_change_size(LOGICAL_REP_MSG_DELETE, + rel->localrel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE); + /* For a partitioned table, apply delete to correct partition. */ if (rel->localrel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE) apply_handle_tuple_routing(edata, @@ -2421,6 +2491,81 @@ apply_dispatch(StringInfo s) } /* + * Subscriber side implementation equivalent to ReorderBufferChangeSize + * of the publisher. + * + * According to the logical replication message type, record major + * resource consumptions of this subscription for each message. + * At present, do not collect data from generic functions to keep + * code simplicity except for tuple length, since the implementation + * complexity versus benefit tradeoff should not be good. + * In terms of tuple length consumption, see logicalrep_read_tuple. + * + * Also, add multiple values at once in order to reduce the number + * of calls to this function, although the disadvantage of this way + * is we cannot get correct transaction size when we get an error. + * + * 'is_partitioned' is used to add some extra size. + */ +static void +update_apply_change_size(LogicalRepMsgType action, bool is_partitioned) +{ + int64 size = 0; + + switch (action) + { + /* No special memory consumption */ + case LOGICAL_REP_MSG_BEGIN: + case LOGICAL_REP_MSG_COMMIT: + case LOGICAL_REP_MSG_TRUNCATE: + case LOGICAL_REP_MSG_TYPE: + case LOGICAL_REP_MSG_ORIGIN: + case LOGICAL_REP_MSG_MESSAGE: + case LOGICAL_REP_MSG_BEGIN_PREPARE: + case LOGICAL_REP_MSG_RELATION: + break; + + case LOGICAL_REP_MSG_INSERT: + case LOGICAL_REP_MSG_UPDATE: + case LOGICAL_REP_MSG_DELETE: + Assert(!in_streamed_transaction); + + /* + * Compute size based on ApplyExecutionData. The size of + * LogicalRepRelMapEntry can be skipped because it is obtained + * from hash_search in logicalrep_rel_open. + */ + size += sizeof(ApplyExecutionData) + sizeof(EState) + + sizeof(ResultRelInfo) + sizeof(ResultRelInfo); + + /* Add some extra size if the target relation is partitioned. */ + if (is_partitioned) + size += sizeof(ModifyTableState) + sizeof(PartitionTupleRouting); + break; + + case LOGICAL_REP_MSG_PREPARE: + case LOGICAL_REP_MSG_COMMIT_PREPARED: + case LOGICAL_REP_MSG_ROLLBACK_PREPARED: + size += sizeof(FlushPosition); + break; + + case LOGICAL_REP_MSG_STREAM_START: + case LOGICAL_REP_MSG_STREAM_STOP: + case LOGICAL_REP_MSG_STREAM_COMMIT: + case LOGICAL_REP_MSG_STREAM_PREPARE: + case LOGICAL_REP_MSG_STREAM_ABORT: + default: + ereport(ERROR, + (errcode(ERRCODE_PROTOCOL_VIOLATION), + errmsg("invalid logical replication message type \"%c\"", action))); + } + + /* Update the total size of consumption when necessary */ + if (size != 0) + add_apply_error_context_xact_size(size); +} + +/* * Figure out which write/flush positions to report to the walsender process. * * We can't simply report back the last LSN the walsender sent us because the @@ -3294,6 +3439,7 @@ stream_write_change(char action, StringInfo s) BufFileWrite(stream_fd, &s->data[s->cursor], len); } + /* * Cleanup the memory for subxacts and reset the related variables. */ @@ -3604,6 +3750,27 @@ ApplyWorkerMain(Datum main_arg) proc_exit(0); } +/* Exported so that stats collector can utilize this value */ +int64 +get_apply_error_context_xact_size(void) +{ + return apply_error_callback_arg.bytes; +} + +/* Add size to apply error bytes */ +void +add_apply_error_context_xact_size(int64 size) +{ + apply_error_callback_arg.bytes += size; +} + +/* Reset information of apply error callback */ +void +reset_apply_error_context_xact_size(void) +{ + apply_error_callback_arg.bytes = 0; +} + /* * Is current process a logical replication worker? */ diff --git a/src/backend/utils/adt/pgstatfuncs.c b/src/backend/utils/adt/pgstatfuncs.c index e7ff4ef..2682316 100644 --- a/src/backend/utils/adt/pgstatfuncs.c +++ b/src/backend/utils/adt/pgstatfuncs.c @@ -2414,7 +2414,7 @@ pg_stat_get_replication_slot(PG_FUNCTION_ARGS) Datum pg_stat_get_subscription_worker(PG_FUNCTION_ARGS) { -#define PG_STAT_GET_SUBSCRIPTION_WORKER_COLS 9 +#define PG_STAT_GET_SUBSCRIPTION_WORKER_COLS 15 Oid subid = PG_GETARG_OID(0); Oid subrelid; TupleDesc tupdesc; @@ -2441,19 +2441,31 @@ pg_stat_get_subscription_worker(PG_FUNCTION_ARGS) OIDOID, -1, 0); TupleDescInitEntry(tupdesc, (AttrNumber) 2, "subrelid", OIDOID, -1, 0); - TupleDescInitEntry(tupdesc, (AttrNumber) 3, "last_error_relid", + TupleDescInitEntry(tupdesc, (AttrNumber) 3, "commit_count", + INT8OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) 4, "commit_bytes", + INT8OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) 5, "error_count", + INT8OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) 6, "error_bytes", + INT8OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) 7, "abort_count", + INT8OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) 8, "abort_bytes", + INT8OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) 9, "last_error_relid", OIDOID, -1, 0); - TupleDescInitEntry(tupdesc, (AttrNumber) 4, "last_error_command", + TupleDescInitEntry(tupdesc, (AttrNumber) 10, "last_error_command", TEXTOID, -1, 0); - TupleDescInitEntry(tupdesc, (AttrNumber) 5, "last_error_xid", + TupleDescInitEntry(tupdesc, (AttrNumber) 11, "last_error_xid", XIDOID, -1, 0); - TupleDescInitEntry(tupdesc, (AttrNumber) 6, "last_error_count", + TupleDescInitEntry(tupdesc, (AttrNumber) 12, "last_error_count", INT8OID, -1, 0); - TupleDescInitEntry(tupdesc, (AttrNumber) 7, "last_error_message", + TupleDescInitEntry(tupdesc, (AttrNumber) 13, "last_error_message", TEXTOID, -1, 0); - TupleDescInitEntry(tupdesc, (AttrNumber) 8, "first_error_time", + TupleDescInitEntry(tupdesc, (AttrNumber) 14, "first_error_time", TIMESTAMPTZOID, -1, 0); - TupleDescInitEntry(tupdesc, (AttrNumber) 9, "last_error_time", + TupleDescInitEntry(tupdesc, (AttrNumber) 15, "last_error_time", TIMESTAMPTZOID, -1, 0); BlessTupleDesc(tupdesc); @@ -2471,6 +2483,14 @@ pg_stat_get_subscription_worker(PG_FUNCTION_ARGS) else nulls[i++] = true; + /* transaction stats */ + values[i++] = Int64GetDatum(wentry->xact_commit_count); + values[i++] = Int64GetDatum(wentry->xact_commit_bytes); + values[i++] = Int64GetDatum(wentry->xact_error_count); + values[i++] = Int64GetDatum(wentry->xact_error_bytes); + values[i++] = Int64GetDatum(wentry->xact_abort_count); + values[i++] = Int64GetDatum(wentry->xact_abort_bytes); + /* last_error_relid */ if (OidIsValid(wentry->relid)) values[i++] = ObjectIdGetDatum(wentry->relid); diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index 25f685f..fc654b4 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -5389,9 +5389,9 @@ proname => 'pg_stat_get_subscription_worker', prorows => '1', proisstrict => 'f', proretset => 't', provolatile => 's', proparallel => 'r', prorettype => 'record', proargtypes => 'oid oid', - proallargtypes => '{oid,oid,oid,oid,oid,text,xid,int8,text,timestamptz,timestamptz}', - proargmodes => '{i,i,o,o,o,o,o,o,o,o,o}', - proargnames => '{subid,subrelid,subid,subrelid,last_error_relid,last_error_command,last_error_xid,last_error_count,last_error_message,first_error_time,last_error_time}', + proallargtypes => '{oid,oid,oid,oid,int8,int8,int8,int8,int8,int8,oid,text,xid,int8,text,timestamptz,timestamptz}', + proargmodes => '{i,i,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o}', + proargnames => '{subid,subrelid,subid,subrelid,commit_count,commit_bytes,error_count,error_bytes,abort_count,abort_bytes,last_error_relid,last_error_command,last_error_xid,last_error_count,last_error_message,first_error_time,last_error_time}', prosrc => 'pg_stat_get_subscription_worker' }, { oid => '6118', descr => 'statistics: information about subscription', proname => 'pg_stat_get_subscription', prorows => '10', proisstrict => 'f', diff --git a/src/include/pgstat.h b/src/include/pgstat.h index 6643938..f01e374 100644 --- a/src/include/pgstat.h +++ b/src/include/pgstat.h @@ -86,6 +86,8 @@ typedef enum StatMsgType PGSTAT_MTYPE_DISCONNECT, PGSTAT_MTYPE_SUBSCRIPTIONPURGE, PGSTAT_MTYPE_SUBWORKERERROR, + PGSTAT_MTYPE_SUBWORKERXACTEND, + PGSTAT_MTYPE_SUBWORKERTWOPHASEXACT, } StatMsgType; /* ---------- @@ -558,6 +560,56 @@ typedef struct PgStat_MsgSubscriptionPurge } PgStat_MsgSubscriptionPurge; /* ---------- + * PgStat_MsgSubscriptionXactEnd Sent by the apply worker or the table sync worker + * to report successful transaction ends. + * ---------- + */ +typedef struct PgStat_MsgSubWorkerXactEnd +{ + PgStat_MsgHdr m_hdr; + + /* determine the worker entry */ + Oid m_databaseid; + Oid m_subid; + Oid m_subrelid; + + /* + * distinguish between transaction commits and streaming transaction + * aborts that are handled without error. + */ + LogicalRepMsgType m_command; + + /* memory consumption used by transaction */ + PgStat_Counter m_xact_bytes; + +} PgStat_MsgSubWorkerXactEnd; + +/* ---------- + * PgStat_MsgSubWorkerTwophaseXact Sent by the apply worker to make size of prepared + * txn persistent over the server restart and make it + * visible after commit prepare or rollback prepared. + * This is separated from PgStat_MsgSubWorkerXactEnd + * so that we can reduce message size of gid for other + * operations (e.g. normal COMMIT) that should happen more + * frequently than prepare operation usually. + * ---------- + */ +typedef struct PgStat_MsgSubWorkerTwophaseXact +{ + PgStat_MsgHdr m_hdr; + + /* determine the subscription */ + Oid m_databaseid; + Oid m_subid; + + LogicalRepMsgType m_command; + char m_gid[GIDSIZE]; + int gid_len; + PgStat_Counter m_xact_bytes; + +} PgStat_MsgSubWorkerTwophaseXact; + +/* ---------- * PgStat_MsgSubWorkerError Sent by the apply worker or the table sync worker to * report the error occurred during logical replication. * ---------- @@ -577,6 +629,12 @@ typedef struct PgStat_MsgSubWorkerError Oid m_subrelid; /* + * Transaction stats of subscription needs to be updated when an error + * occurs. + */ + PgStat_Counter m_xact_error_bytes; + + /* * Oids of the database and the table that the reporter was actually * processing. m_relid can be InvalidOid if an error occurred during * worker applying a non-data-modification message such as RELATION. @@ -768,6 +826,8 @@ typedef union PgStat_Msg PgStat_MsgDisconnect msg_disconnect; PgStat_MsgSubscriptionPurge msg_subscriptionpurge; PgStat_MsgSubWorkerError msg_subworkererror; + PgStat_MsgSubWorkerXactEnd msg_subworkerxactend; + PgStat_MsgSubWorkerTwophaseXact msg_subworkertwophasexact; } PgStat_Msg; @@ -822,16 +882,23 @@ typedef struct PgStat_StatDBEntry TimestampTz stats_timestamp; /* time of db stats file update */ /* - * tables, functions, and subscription workers must be last in the struct, - * because we don't write the pointers out to the stats file. + * tables, functions, subscription workers and its prepared transaction + * stats must be last in the struct, because we don't write the pointers + * out to the stats file. * * subworker is the hash table of PgStat_StatSubWorkerEntry which stores - * statistics of logical replication workers: apply worker - * and table sync worker. + * statistics of logical replication workers: apply worker and table sync + * worker. + * + * subworkers_preparedsizes should give appropriate transaction sizes at + * either commit prepared or rollback prepared time, even when it's after + * the server restart. We have the apply worker send those statistics to + * the stats collector at prepare time. */ HTAB *tables; HTAB *functions; HTAB *subworkers; + HTAB *subworkers_preparedsizes; } PgStat_StatDBEntry; @@ -1005,6 +1072,16 @@ typedef struct PgStat_StatSubWorkerEntry PgStat_StatSubWorkerKey key; /* hash key (must be first) */ /* + * Cumulative transaction statistics of subscription worker + */ + PgStat_Counter xact_commit_count; + PgStat_Counter xact_commit_bytes; + PgStat_Counter xact_error_count; + PgStat_Counter xact_error_bytes; + PgStat_Counter xact_abort_count; + PgStat_Counter xact_abort_bytes; + + /* * Subscription worker error statistics representing an error that * occurred during application of logical replication or the initial table * synchronization. @@ -1019,6 +1096,22 @@ typedef struct PgStat_StatSubWorkerEntry char error_message[PGSTAT_SUBWORKERERROR_MSGLEN]; } PgStat_StatSubWorkerEntry; +/* prepared transaction */ +typedef struct PgStat_SW_PreparedXactKey +{ + Oid subid; + char gid[GIDSIZE]; +} PgStat_SW_PreparedXactKey; + +typedef struct PgStat_SW_PreparedXactEntry +{ + PgStat_SW_PreparedXactKey key; /* hash key */ + + Oid subid; + char gid[GIDSIZE]; + PgStat_Counter xact_size; +} PgStat_SW_PreparedXactEntry; + /* * Working state needed to accumulate per-function-call timing statistics. */ @@ -1129,6 +1222,10 @@ extern void pgstat_report_checksum_failure(void); extern void pgstat_report_replslot(const PgStat_StatReplSlotEntry *repSlotStat); extern void pgstat_report_replslot_create(const char *slotname); extern void pgstat_report_replslot_drop(const char *slotname); +extern void pgstat_report_subworker_xact_end(Oid subid, Oid subrel, + LogicalRepMsgType command, PgStat_Counter xact_size); +extern void pgstat_report_subworker_twophase_xact(Oid subid, LogicalRepMsgType command, + PgStat_Counter xact_size, char *gid); extern void pgstat_report_subworker_error(Oid subid, Oid subrelid, Oid relid, LogicalRepMsgType command, TransactionId xid, const char *errmsg); diff --git a/src/include/replication/logicalworker.h b/src/include/replication/logicalworker.h index 2ad61a0..9a8447b 100644 --- a/src/include/replication/logicalworker.h +++ b/src/include/replication/logicalworker.h @@ -16,4 +16,9 @@ extern void ApplyWorkerMain(Datum main_arg); extern bool IsLogicalWorker(void); +/* for transaction stats */ +extern int64 get_apply_error_context_xact_size(void); +extern void add_apply_error_context_xact_size(int64 size); +extern void reset_apply_error_context_xact_size(void); + #endif /* LOGICALWORKER_H */ diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out index d60c5a5..bd3e221 100644 --- a/src/test/regress/expected/rules.out +++ b/src/test/regress/expected/rules.out @@ -2097,6 +2097,12 @@ pg_stat_subscription| SELECT su.oid AS subid, pg_stat_subscription_workers| SELECT w.subid, s.subname, w.subrelid, + w.commit_count, + w.commit_bytes, + w.error_count, + w.error_bytes, + w.abort_count, + w.abort_bytes, w.last_error_relid, w.last_error_command, w.last_error_xid, @@ -2112,7 +2118,7 @@ pg_stat_subscription_workers| SELECT w.subid, pg_subscription_rel.srrelid AS relid FROM pg_subscription_rel WHERE (pg_subscription_rel.srsubstate <> 'r'::"char")) sr, - (LATERAL pg_stat_get_subscription_worker(sr.subid, sr.relid) w(subid, subrelid, last_error_relid, last_error_command, last_error_xid, last_error_count, last_error_message, first_error_time, last_error_time) + (LATERAL pg_stat_get_subscription_worker(sr.subid, sr.relid) w(subid, subrelid, commit_count, commit_bytes, error_count, error_bytes, abort_count, abort_bytes, last_error_relid, last_error_command, last_error_xid, last_error_count, last_error_message, first_error_time, last_error_time) JOIN pg_subscription s ON ((w.subid = s.oid))); pg_stat_sys_indexes| SELECT pg_stat_all_indexes.relid, pg_stat_all_indexes.indexrelid, diff --git a/src/test/subscription/t/027_worker_xact_stats.pl b/src/test/subscription/t/027_worker_xact_stats.pl new file mode 100644 index 0000000..e09be42 --- /dev/null +++ b/src/test/subscription/t/027_worker_xact_stats.pl @@ -0,0 +1,146 @@ + +# Copyright (c) 2021, PostgreSQL Global Development Group + +# Tests for subscription worker statistics during apply. +use strict; +use warnings; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More tests => 2; + +# Create publisher node +my $node_publisher = PostgreSQL::Test::Cluster->new('publisher'); +$node_publisher->init(allows_streaming => 'logical'); +$node_publisher->append_conf( + 'postgresql.conf', qq[ +logical_decoding_work_mem = 64kB +max_prepared_transactions = 10 +max_wal_senders = 10 +wal_sender_timeout = 0 +]); +$node_publisher->start; + +# Create subscriber node +my $node_subscriber = PostgreSQL::Test::Cluster->new('subscriber'); +$node_subscriber->init(allows_streaming => 'logical'); +$node_subscriber->append_conf( + 'postgresql.conf', qq[ +max_prepared_transactions = 10 +log_min_messages = DEBUG1 +]); +$node_subscriber->start; + +# Setup structures on the publisher and the subscriber +$node_publisher->safe_psql('postgres', "CREATE TABLE test_tab (a int)"); +$node_subscriber->safe_psql('postgres', + "CREATE TABLE test_tab (a int primary key);"); + +# Setup logical replication +my $publisher_connstr = $node_publisher->connstr . ' dbname=postgres'; +$node_publisher->safe_psql('postgres', + "CREATE PUBLICATION tap_pub FOR TABLE test_tab"); + +my $appname = 'tap_sub'; +my $subopts = 'streaming = on, two_phase = on, copy_data = false'; +$node_subscriber->safe_psql( + 'postgres', qq[ +CREATE SUBSCRIPTION tap_sub +CONNECTION '$publisher_connstr application_name=$appname' +PUBLICATION tap_pub WITH ($subopts); +]); + +$node_publisher->wait_for_catchup($appname); + +# There's no entry at the beginning +my $result = $node_subscriber->safe_psql('postgres', + "SELECT count(*) FROM pg_stat_subscription_workers;"); +is($result, q(0), 'no entry for transaction stats yet'); + +# COMMIT +$node_publisher->safe_psql('postgres', + "BEGIN; INSERT INTO test_tab VALUES (1); COMMIT;"); + +$node_subscriber->poll_query_until('postgres', + "SELECT count(1) = 1 FROM pg_stat_subscription_workers where commit_count = 1;" +) or die "didn't get updates of xact stats by commit"; + +# Now, stats collector make the bytes updated also. +$result = $node_subscriber->safe_psql('postgres', + "SELECT commit_bytes > 0 FROM pg_stat_subscription_workers;"); +is($result, q(t), 'got consumed bytes'); + +# STREAM COMMIT +$node_publisher->safe_psql('postgres', + "BEGIN; INSERT INTO test_tab VALUES(generate_series(1001, 2000)); COMMIT;" +); +$node_subscriber->poll_query_until('postgres', + "SELECT count(1) = 1 FROM pg_stat_subscription_workers where commit_count = 2;" +) or die "didn't get updates of xact stats by stream commit"; + +# STREAM PREPARE & COMMIT PREPARED +# This should restore the xact size before the shutdown. +$node_publisher->safe_psql( + 'postgres', q[ +BEGIN; +INSERT INTO test_tab VALUES(generate_series(2001, 3000)); +PREPARE TRANSACTION 'gid1'; +]); + +# This streamed prepare is not displayed until the commit prepared +# or rollback prepared. Hence, there's no way to confirm that +# stats collector has received the bytes of prepared transaction. +# So, instead of checking the view, issue one more committed transaction +# after the prepare and make sure that this commit's update is done, +# which should mean the previous streamed prepare is already processed +# by the stats collector as well. +$node_publisher->safe_psql('postgres', + "BEGIN; INSERT INTO test_tab VALUES (2); COMMIT;"); +$node_subscriber->poll_query_until('postgres', + "SELECT count(1) = 1 FROM pg_stat_subscription_workers where commit_count = 3;" +) or die "didn't process the updates of committed transaction"; + +$node_subscriber->restart; + +# Get commit_bytes before commit prepared. +my $tmp = $node_subscriber->safe_psql('postgres', + "SELECT commit_bytes FROM pg_stat_subscription_workers where commit_count = 3;" +); + +# Commit prepared increments the commit_count. +$node_publisher->safe_psql('postgres', "COMMIT PREPARED 'gid1'"); +$node_subscriber->poll_query_until('postgres', + "SELECT count(1) = 1 FROM pg_stat_subscription_workers where commit_count = 4;" + ) + or die + "didn't get updates of xact stats by stream prepare and commit prepared"; + +$node_subscriber->poll_query_until('postgres', + "SELECT commit_bytes > $tmp FROM pg_stat_subscription_workers where commit_count = 4;" +); + +# ROLLBACK PREPARED +$node_publisher->safe_psql( + 'postgres', q[ +BEGIN; +INSERT INTO test_tab VALUES (3); +PREPARE TRANSACTION 'gid2'; +]); +$node_publisher->safe_psql('postgres', "ROLLBACK PREPARED 'gid2'"); +$node_subscriber->poll_query_until('postgres', + "SELECT count(1) = 1 FROM pg_stat_subscription_workers where abort_count = 1;" +) or die "didn't get updates of xact stats by rollback prepared"; + +# error stats (by duplication error) +$node_publisher->safe_psql( + 'postgres', q[ +BEGIN; +INSERT INTO test_tab VALUES (1); +COMMIT; +]); + +$node_subscriber->poll_query_until('postgres', + "SELECT count(1) = 1 FROM pg_stat_subscription_workers where error_count > 0;" +) or die "didn't get updates of xact stats by error"; + +$node_subscriber->stop('fast'); +$node_publisher->stop('fast'); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f41ef0d..3403667 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -1945,6 +1945,8 @@ PgStat_MsgResetslrucounter PgStat_MsgSLRU PgStat_MsgSubscriptionPurge PgStat_MsgSubWorkerError +PgStat_MsgSubWorkerTwophaseXact +PgStat_MsgSubWorkerXactEnd PgStat_MsgTabpurge PgStat_MsgTabstat PgStat_MsgTempFile @@ -1958,6 +1960,8 @@ PgStat_StatFuncEntry PgStat_StatReplSlotEntry PgStat_StatSubWorkerEntry PgStat_StatSubWorkerKey +PgStat_SW_PreparedXactKey +PgStat_SW_PreparedXactEntry PgStat_StatTabEntry PgStat_SubXactStatus PgStat_TableCounts -- 2.2.0 ^ permalink raw reply [nested|flat] 113+ messages in thread
* RE: Failed transaction statistics to measure the logical replication progress @ 2021-11-15 09:31 [email protected] <[email protected]> parent: vignesh C <[email protected]> 1 sibling, 0 replies; 113+ messages in thread From: [email protected] @ 2021-11-15 09:31 UTC (permalink / raw) To: 'vignesh C' <[email protected]>; +Cc: Greg Nancarrow <[email protected]>; Masahiko Sawada <[email protected]>; Amit Kapila <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]> On Wednesday, November 10, 2021 7:13 PM vignesh C <[email protected]> wrote: > On Tue, Nov 9, 2021 at 5:05 PM [email protected] > <[email protected]> wrote: > > Yes. I've rebased and updated the patch, paying attention to this point. > > Attached the updated version. > > Thanks for the updated patch, few comments: > 1) you could rename PgStat_StatSubWorkerPreparedXact to > PgStat_SW_PreparedXactKey or a simpler name which includes key and > similarly change PgStat_StatSubWorkerPreparedXactSize to > PgStat_SW_PreparedXactEntry > > +/* prepared transaction */ > +typedef struct PgStat_StatSubWorkerPreparedXact { > + Oid subid; > + char gid[GIDSIZE]; > +} PgStat_StatSubWorkerPreparedXact; > + > +typedef struct PgStat_StatSubWorkerPreparedXactSize > +{ > + PgStat_StatSubWorkerPreparedXact key; /* hash key */ > + > + Oid subid; > + char gid[GIDSIZE]; > + PgStat_Counter xact_size; > +} PgStat_StatSubWorkerPreparedXactSize; > + Fixed. Adopted your suggested names. > 2) You can change prepared_size to sw_prepared_xact_entry or > prepared_xact_entry since it is a hash entry with few fields > + if (subWorkerPreparedXactSizeHash) > + { > + PgStat_StatSubWorkerPreparedXactSize *prepared_size; > + > + hash_seq_init(&hstat, subWorkerPreparedXactSizeHash); > + while((prepared_size = > (PgStat_StatSubWorkerPreparedXactSize *) hash_seq_search(&hstat)) != > NULL) > + { > + fputc('P', fpout); > + rc = fwrite(prepared_size, > sizeof(PgStat_StatSubWorkerPreparedXactSize), 1, fpout); > + (void) rc; /* we'll check > for error with ferror */ > + } I preferred prepared_xact_entry. Fixed. > 3) This need to be indented > - w.relid, > - w.command, > - w.xid, > - w.error_count, > - w.error_message, > - w.last_error_time > + w.commit_count, > + w.commit_bytes, > + w.error_count, > + w.error_bytes, > + w.abort_count, > + w.abort_bytes, > + w.last_error_relid, > + w.last_error_command, > + w.last_error_xid, > + w.last_error_count, > + w.last_error_message, > + w.last_error_time Fixed. > 4) Instead of adding a function to calculate the size, can we move > PartitionTupleRouting from c file to the header file and use sizeof at the caller > function? > +/* > + * PartitionTupleRoutingSize - exported to calculate total data size > + * of logical replication mesage apply, because this is one of the > + * ApplyExecutionData struct members. > + */ > +size_t > +PartitionTupleRoutingSize(void) > +{ > + return sizeof(PartitionTupleRouting); } Fixed. > 5) You could run pgindent and pgperltidy for the code and test code to fix the > indent issues. > + > subWorkerPreparedXactSizeHash = hash_create("Subscription worker stats > of prepared txn", > + > > PGSTAT_SUBWORKER_HASH_SIZE, > + > &hash_ctl, > + > HASH_ELEM | HASH_STRINGS | > HASH_CONTEXT); > +# There's no entry at the beginning > +my $result = $node_subscriber->safe_psql('postgres', > +"SELECT count(*) FROM pg_stat_subscription_workers;"); is($result, > +q(0), 'no entry for transaction stats yet'); Conducted pgindent and pgperltidy. > 6) Few places you have used strlcpy and few places you have used memcpy, > you can keep it consistent: > + msg.m_command = command; > + strlcpy(msg.m_gid, gid, sizeof(msg.m_gid)); > + msg.m_xact_bytes = xact_size; > > + key.subid = subid; > + memcpy(key.gid, gid, sizeof(key.gid)); > + action = (create ? HASH_ENTER : HASH_FIND); Fixed. I used strlcpy for new additional functions I made. An exception is pgstat_read_db_statsfile(). In this function, we use memcpy() consistently in other places. Please have a look at [1] [1] - https://www.postgresql.org/message-id/TYCPR01MB8373FEB287F733C81C1E4D42ED989%40TYCPR01MB8373.jpnprd0... Best Regards, Takamichi Osumi ^ permalink raw reply [nested|flat] 113+ messages in thread
* RE: Failed transaction statistics to measure the logical replication progress @ 2021-11-15 09:32 [email protected] <[email protected]> parent: [email protected] <[email protected]> 0 siblings, 0 replies; 113+ messages in thread From: [email protected] @ 2021-11-15 09:32 UTC (permalink / raw) To: 'Dilip Kumar' <[email protected]>; +Cc: Greg Nancarrow <[email protected]>; Masahiko Sawada <[email protected]>; Amit Kapila <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]> On Wednesday, November 10, 2021 6:13 PM I wrote: > On Wednesday, November 10, 2021 3:43 PM Dilip Kumar > <[email protected]> wrote: > > On Tue, Nov 9, 2021 at 5:05 PM [email protected] > > <[email protected]> wrote: > > > On Tuesday, November 9, 2021 12:08 PM Greg Nancarrow > > <[email protected]> wrote: > > > > On Fri, Nov 5, 2021 at 7:11 PM [email protected] > > > > <[email protected]> wrote: > > > > > > > > > > > > > I did a quick scan through the latest v8 patch and noticed the > > > > following > > things: > > > I appreciate your review ! > > I have reviewed some part of the patch and I have a few comments > I really appreciate your attention and review. ... > > 3. > > + { > > + size += *extra_data->stream_write_len; > > + add_apply_error_context_xact_size(size); > > + return; > > + } > > > > From apply_handle_insert(), we are calling update_apply_change_size(), > > and inside this function we are dereferencing > *extra_data->stream_write_len. > > Basically, stream_write_len is in integer pointer and the caller > > hasn't allocated memory for that and inside update_apply_change_size, > > we are directly dereferencing the pointer, how this can be correct. ... > I'll just delete the top part that handles streaming bytes calculation in the > update_apply_change_size(). > It's because now that there is a specific structure to recognize each streaming > xid and save transaction size there, which makes the top part in question > useless. > > > And I also see that in the > > whole patch stream_write_len, is never used as lvalue so without > > storing anything into this why are we trying to use this as rvalue > > here? This is clearly an issue. > As described above, I'll fix this part and related codes mainly streaming related > codes in the next version. Removed this deadcodes. Please have a look at [1] [1] - https://www.postgresql.org/message-id/TYCPR01MB8373FEB287F733C81C1E4D42ED989%40TYCPR01MB8373.jpnprd0... Best Regards, Takamichi Osumi ^ permalink raw reply [nested|flat] 113+ messages in thread
* RE: Failed transaction statistics to measure the logical replication progress @ 2021-11-15 12:13 [email protected] <[email protected]> parent: [email protected] <[email protected]> 0 siblings, 1 reply; 113+ messages in thread From: [email protected] @ 2021-11-15 12:13 UTC (permalink / raw) To: 'vignesh C' <[email protected]>; +Cc: Greg Nancarrow <[email protected]>; Masahiko Sawada <[email protected]>; Amit Kapila <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]> On Monday, November 15, 2021 6:28 PM I wrote: > On Thursday, November 11, 2021 9:47 PM vignesh C <[email protected]> > wrote: > > Few more comments: ... > Fixed. Also, its name was too long > when aligned with other PgStat_StatDBEntry memebers. > Thus I renamed it as subworkers_preparedsizes. > > This depends on v21 in [1] Hi There was some minor conflict error by the change of v22 in [1]. I've conducted some update for this. (The rebased part is only C code and checked by pgindent) [1] - https://www.postgresql.org/message-id/CAD21AoCE72vKXJp99f4xRw7Mh5ve-Z2roe21gP8Y82_CxXKvbg%40mail.gma... Best Regards, Takamichi Osumi Attachments: [application/octet-stream] v11-0001-Rename-existing-columns-of-pg_stat_subscription_.patch (15.4K, ../../TYCPR01MB8373D7AE0B6C62E5CDB03C6CED989@TYCPR01MB8373.jpnprd01.prod.outlook.com/2-v11-0001-Rename-existing-columns-of-pg_stat_subscription_.patch) download | inline diff: From bb6d889e1bb985a24cdc8532f3779ba83e582c70 Mon Sep 17 00:00:00 2001 From: Osumi Takamichi <[email protected]> Date: Mon, 15 Nov 2021 08:45:55 +0000 Subject: [PATCH v11 1/2] Rename existing columns of pg_stat_subscription_workers and export PartitionTupleRouting --- doc/src/sgml/monitoring.sgml | 24 +++++----- src/backend/catalog/system_views.sql | 10 ++--- src/backend/executor/execPartition.c | 70 ----------------------------- src/backend/utils/adt/pgstatfuncs.c | 20 ++++----- src/include/catalog/pg_proc.dat | 2 +- src/include/executor/execPartition.h | 69 ++++++++++++++++++++++++++++ src/test/regress/expected/rules.out | 12 ++--- src/test/subscription/t/026_error_report.pl | 8 ++-- 8 files changed, 107 insertions(+), 108 deletions(-) diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml index daf9fe8..5b10d18 100644 --- a/doc/src/sgml/monitoring.sgml +++ b/doc/src/sgml/monitoring.sgml @@ -3103,31 +3103,31 @@ SELECT pid, wait_event_type, wait_event FROM pg_stat_activity WHERE wait_event i <row> <entry role="catalog_table_entry"><para role="column_definition"> - <structfield>relid</structfield> <type>oid</type> + <structfield>last_error_relid</structfield> <type>oid</type> </para> <para> OID of the relation that the worker was processing when the - error occurred + last error occurred </para></entry> </row> <row> <entry role="catalog_table_entry"><para role="column_definition"> - <structfield>command</structfield> <type>text</type> + <structfield>last_error_command</structfield> <type>text</type> </para> <para> - Name of command being applied when the error occurred. This field - is always NULL if the error was reported during the initial data - copy. + Name of the last command being applied when the error occurred. + This field is always NULL if the error was reported during the + initial data copy. </para></entry> </row> <row> <entry role="catalog_table_entry"><para role="column_definition"> - <structfield>xid</structfield> <type>xid</type> + <structfield>last_error_xid</structfield> <type>xid</type> </para> <para> - Transaction ID of the publisher node being applied when the error + Transaction ID of the publisher node being applied when the last error occurred. This field is always NULL if the error was reported during the initial data copy. </para></entry> @@ -3135,19 +3135,19 @@ SELECT pid, wait_event_type, wait_event FROM pg_stat_activity WHERE wait_event i <row> <entry role="catalog_table_entry"><para role="column_definition"> - <structfield>error_count</structfield> <type>uint8</type> + <structfield>last_error_count</structfield> <type>uint8</type> </para> <para> - Number of consecutive times the error occurred + Number of consecutive times the last error occurred </para></entry> </row> <row> <entry role="catalog_table_entry"><para role="column_definition"> - <structfield>error_message</structfield> <type>text</type> + <structfield>last_error_message</structfield> <type>text</type> </para> <para> - The error message + The last error message </para></entry> </row> diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index cb2f77c..ecf1a0b 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1267,11 +1267,11 @@ CREATE VIEW pg_stat_subscription_workers AS w.subid, s.subname, w.subrelid, - w.relid, - w.command, - w.xid, - w.error_count, - w.error_message, + w.last_error_relid, + w.last_error_command, + w.last_error_xid, + w.last_error_count, + w.last_error_message, w.first_error_time, w.last_error_time FROM (SELECT diff --git a/src/backend/executor/execPartition.c b/src/backend/executor/execPartition.c index 5c723bc..2467f4c 100644 --- a/src/backend/executor/execPartition.c +++ b/src/backend/executor/execPartition.c @@ -34,76 +34,6 @@ #include "utils/rls.h" #include "utils/ruleutils.h" - -/*----------------------- - * PartitionTupleRouting - Encapsulates all information required to - * route a tuple inserted into a partitioned table to one of its leaf - * partitions. - * - * partition_root - * The partitioned table that's the target of the command. - * - * partition_dispatch_info - * Array of 'max_dispatch' elements containing a pointer to a - * PartitionDispatch object for every partitioned table touched by tuple - * routing. The entry for the target partitioned table is *always* - * present in the 0th element of this array. See comment for - * PartitionDispatchData->indexes for details on how this array is - * indexed. - * - * nonleaf_partitions - * Array of 'max_dispatch' elements containing pointers to fake - * ResultRelInfo objects for nonleaf partitions, useful for checking - * the partition constraint. - * - * num_dispatch - * The current number of items stored in the 'partition_dispatch_info' - * array. Also serves as the index of the next free array element for - * new PartitionDispatch objects that need to be stored. - * - * max_dispatch - * The current allocated size of the 'partition_dispatch_info' array. - * - * partitions - * Array of 'max_partitions' elements containing a pointer to a - * ResultRelInfo for every leaf partition touched by tuple routing. - * Some of these are pointers to ResultRelInfos which are borrowed out of - * the owning ModifyTableState node. The remainder have been built - * especially for tuple routing. See comment for - * PartitionDispatchData->indexes for details on how this array is - * indexed. - * - * is_borrowed_rel - * Array of 'max_partitions' booleans recording whether a given entry - * in 'partitions' is a ResultRelInfo pointer borrowed from the owning - * ModifyTableState node, rather than being built here. - * - * num_partitions - * The current number of items stored in the 'partitions' array. Also - * serves as the index of the next free array element for new - * ResultRelInfo objects that need to be stored. - * - * max_partitions - * The current allocated size of the 'partitions' array. - * - * memcxt - * Memory context used to allocate subsidiary structs. - *----------------------- - */ -struct PartitionTupleRouting -{ - Relation partition_root; - PartitionDispatch *partition_dispatch_info; - ResultRelInfo **nonleaf_partitions; - int num_dispatch; - int max_dispatch; - ResultRelInfo **partitions; - bool *is_borrowed_rel; - int num_partitions; - int max_partitions; - MemoryContext memcxt; -}; - /*----------------------- * PartitionDispatch - information about one partitioned table in a partition * hierarchy required to route a tuple to any of its partitions. A diff --git a/src/backend/utils/adt/pgstatfuncs.c b/src/backend/utils/adt/pgstatfuncs.c index b19729d..a34c0b6 100644 --- a/src/backend/utils/adt/pgstatfuncs.c +++ b/src/backend/utils/adt/pgstatfuncs.c @@ -2441,15 +2441,15 @@ pg_stat_get_subscription_worker(PG_FUNCTION_ARGS) OIDOID, -1, 0); TupleDescInitEntry(tupdesc, (AttrNumber) 2, "subrelid", OIDOID, -1, 0); - TupleDescInitEntry(tupdesc, (AttrNumber) 3, "relid", + TupleDescInitEntry(tupdesc, (AttrNumber) 3, "last_error_relid", OIDOID, -1, 0); - TupleDescInitEntry(tupdesc, (AttrNumber) 4, "command", + TupleDescInitEntry(tupdesc, (AttrNumber) 4, "last_error_command", TEXTOID, -1, 0); - TupleDescInitEntry(tupdesc, (AttrNumber) 5, "xid", + TupleDescInitEntry(tupdesc, (AttrNumber) 5, "last_error_xid", XIDOID, -1, 0); - TupleDescInitEntry(tupdesc, (AttrNumber) 6, "error_count", + TupleDescInitEntry(tupdesc, (AttrNumber) 6, "last_error_count", INT8OID, -1, 0); - TupleDescInitEntry(tupdesc, (AttrNumber) 7, "error_message", + TupleDescInitEntry(tupdesc, (AttrNumber) 7, "last_error_message", TEXTOID, -1, 0); TupleDescInitEntry(tupdesc, (AttrNumber) 8, "first_error_time", TIMESTAMPTZOID, -1, 0); @@ -2471,28 +2471,28 @@ pg_stat_get_subscription_worker(PG_FUNCTION_ARGS) else nulls[i++] = true; - /* relid */ + /* last_error_relid */ if (OidIsValid(wentry->relid)) values[i++] = ObjectIdGetDatum(wentry->relid); else nulls[i++] = true; - /* command */ + /* last_error_command */ if (wentry->command != 0) values[i++] = CStringGetTextDatum(logicalrep_message_type(wentry->command)); else nulls[i++] = true; - /* xid */ + /* last_error_xid */ if (TransactionIdIsValid(wentry->xid)) values[i++] = TransactionIdGetDatum(wentry->xid); else nulls[i++] = true; - /* error_count */ + /* last_error_count */ values[i++] = Int64GetDatum(wentry->error_count); - /* error_message */ + /* last_error_message */ values[i++] = CStringGetTextDatum(wentry->error_message); /* first_error_time */ diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index 50e1c7b..25f685f 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -5391,7 +5391,7 @@ prorettype => 'record', proargtypes => 'oid oid', proallargtypes => '{oid,oid,oid,oid,oid,text,xid,int8,text,timestamptz,timestamptz}', proargmodes => '{i,i,o,o,o,o,o,o,o,o,o}', - proargnames => '{subid,subrelid,subid,subrelid,relid,command,xid,error_count,error_message,first_error_time,last_error_time}', + proargnames => '{subid,subrelid,subid,subrelid,last_error_relid,last_error_command,last_error_xid,last_error_count,last_error_message,first_error_time,last_error_time}', prosrc => 'pg_stat_get_subscription_worker' }, { oid => '6118', descr => 'statistics: information about subscription', proname => 'pg_stat_get_subscription', prorows => '10', proisstrict => 'f', diff --git a/src/include/executor/execPartition.h b/src/include/executor/execPartition.h index 694e38b..4c796ab 100644 --- a/src/include/executor/execPartition.h +++ b/src/include/executor/execPartition.h @@ -110,6 +110,75 @@ typedef struct PartitionPruneState PartitionPruningData *partprunedata[FLEXIBLE_ARRAY_MEMBER]; } PartitionPruneState; +/*----------------------- + * PartitionTupleRouting - Encapsulates all information required to + * route a tuple inserted into a partitioned table to one of its leaf + * partitions. + * + * partition_root + * The partitioned table that's the target of the command. + * + * partition_dispatch_info + * Array of 'max_dispatch' elements containing a pointer to a + * PartitionDispatch object for every partitioned table touched by tuple + * routing. The entry for the target partitioned table is *always* + * present in the 0th element of this array. See comment for + * PartitionDispatchData->indexes for details on how this array is + * indexed. + * + * nonleaf_partitions + * Array of 'max_dispatch' elements containing pointers to fake + * ResultRelInfo objects for nonleaf partitions, useful for checking + * the partition constraint. + * + * num_dispatch + * The current number of items stored in the 'partition_dispatch_info' + * array. Also serves as the index of the next free array element for + * new PartitionDispatch objects that need to be stored. + * + * max_dispatch + * The current allocated size of the 'partition_dispatch_info' array. + * + * partitions + * Array of 'max_partitions' elements containing a pointer to a + * ResultRelInfo for every leaf partition touched by tuple routing. + * Some of these are pointers to ResultRelInfos which are borrowed out of + * the owning ModifyTableState node. The remainder have been built + * especially for tuple routing. See comment for + * PartitionDispatchData->indexes for details on how this array is + * indexed. + * + * is_borrowed_rel + * Array of 'max_partitions' booleans recording whether a given entry + * in 'partitions' is a ResultRelInfo pointer borrowed from the owning + * ModifyTableState node, rather than being built here. + * + * num_partitions + * The current number of items stored in the 'partitions' array. Also + * serves as the index of the next free array element for new + * ResultRelInfo objects that need to be stored. + * + * max_partitions + * The current allocated size of the 'partitions' array. + * + * memcxt + * Memory context used to allocate subsidiary structs. + *----------------------- + */ +struct PartitionTupleRouting +{ + Relation partition_root; + PartitionDispatch *partition_dispatch_info; + ResultRelInfo **nonleaf_partitions; + int num_dispatch; + int max_dispatch; + ResultRelInfo **partitions; + bool *is_borrowed_rel; + int num_partitions; + int max_partitions; + MemoryContext memcxt; +}; + extern PartitionTupleRouting *ExecSetupPartitionTupleRouting(EState *estate, Relation rel); extern ResultRelInfo *ExecFindPartition(ModifyTableState *mtstate, diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out index cb6da2c..d60c5a5 100644 --- a/src/test/regress/expected/rules.out +++ b/src/test/regress/expected/rules.out @@ -2097,11 +2097,11 @@ pg_stat_subscription| SELECT su.oid AS subid, pg_stat_subscription_workers| SELECT w.subid, s.subname, w.subrelid, - w.relid, - w.command, - w.xid, - w.error_count, - w.error_message, + w.last_error_relid, + w.last_error_command, + w.last_error_xid, + w.last_error_count, + w.last_error_message, w.first_error_time, w.last_error_time FROM ( SELECT pg_subscription.oid AS subid, @@ -2112,7 +2112,7 @@ pg_stat_subscription_workers| SELECT w.subid, pg_subscription_rel.srrelid AS relid FROM pg_subscription_rel WHERE (pg_subscription_rel.srsubstate <> 'r'::"char")) sr, - (LATERAL pg_stat_get_subscription_worker(sr.subid, sr.relid) w(subid, subrelid, relid, command, xid, error_count, error_message, first_error_time, last_error_time) + (LATERAL pg_stat_get_subscription_worker(sr.subid, sr.relid) w(subid, subrelid, last_error_relid, last_error_command, last_error_xid, last_error_count, last_error_message, first_error_time, last_error_time) JOIN pg_subscription s ON ((w.subid = s.oid))); pg_stat_sys_indexes| SELECT pg_stat_all_indexes.relid, pg_stat_all_indexes.indexrelid, diff --git a/src/test/subscription/t/026_error_report.pl b/src/test/subscription/t/026_error_report.pl index 1227654..ca30ab2 100644 --- a/src/test/subscription/t/026_error_report.pl +++ b/src/test/subscription/t/026_error_report.pl @@ -15,8 +15,8 @@ sub test_subscription_error my $check_sql = qq[ SELECT count(1) > 0 FROM pg_stat_subscription_workers -WHERE relid = '$relname'::regclass]; - $check_sql .= " AND xid = '$xid'::xid;" if $xid ne ''; +WHERE last_error_relid = '$relname'::regclass]; + $check_sql .= " AND last_error_xid = '$xid'::xid;" if $xid ne ''; # Wait for the error statistics to be updated. $node->poll_query_until( @@ -26,9 +26,9 @@ WHERE relid = '$relname'::regclass]; my $result = $node->safe_psql( 'postgres', qq[ -SELECT subname, command, relid::regclass, error_count > 0 +SELECT subname, last_error_command, last_error_relid::regclass, last_error_count > 0 FROM pg_stat_subscription_workers -WHERE relid = '$relname'::regclass; +WHERE last_error_relid = '$relname'::regclass; ]); is($result, $expected_error, $msg); } -- 2.2.0 [application/octet-stream] v11-0002-Extend-pg_stat_subscription_workers-to-include-g.patch (53.1K, ../../TYCPR01MB8373D7AE0B6C62E5CDB03C6CED989@TYCPR01MB8373.jpnprd01.prod.outlook.com/3-v11-0002-Extend-pg_stat_subscription_workers-to-include-g.patch) download | inline diff: From 5a94939e91e3b215f2eaafea01b036eadddcee21 Mon Sep 17 00:00:00 2001 From: Osumi Takamichi <[email protected]> Date: Mon, 15 Nov 2021 11:44:53 +0000 Subject: [PATCH v11 2/2] Extend pg_stat_subscription_workers to include general transaction statistics Categorize transactions of logical replication subscriber into three types (commit, error, abort) and introduce cumulative columns of those numbers and amounts of consumed data during message apply respectively. The calculation of consumed resources by subscriber's apply is computed based on the data structure for message apply and extra data, which is different from that of publisher's decoding processing. At present, there's no special consideration to spool file statistics such as amount of data spooled to disk or its count. But, this can be added later. The amount of data used by apply for STREAM COMMIT and STREAM PREPARE is transaction data consumption and can be regarded as data that should be added to column of this commit. This size of prepared transaction becomes persistent to conclude the appropriate category for the transaction at either commit prepared or rollback prepared time. Author: Takamichi Osumi Discussed & Reviewed-by: Amit Kapila, Masahiko Sawada, Hou Zhijie, Greg Nancarrow, Vignesh C, Ajin Cherian Discussion: https://www.postgresql.org/message-id/OSBPR01MB48887CA8F40C8D984A6DC00CED199%40OSBPR01MB4888.jpnprd01.prod.outlook.com --- doc/src/sgml/monitoring.sgml | 71 ++++- src/backend/catalog/system_views.sql | 6 + src/backend/postmaster/pgstat.c | 324 ++++++++++++++++++++++- src/backend/replication/logical/proto.c | 12 + src/backend/replication/logical/tablesync.c | 6 + src/backend/replication/logical/worker.c | 169 +++++++++++- src/backend/utils/adt/pgstatfuncs.c | 36 ++- src/include/catalog/pg_proc.dat | 6 +- src/include/pgstat.h | 101 ++++++- src/include/replication/logicalworker.h | 5 + src/test/regress/expected/rules.out | 8 +- src/test/subscription/t/027_worker_xact_stats.pl | 146 ++++++++++ src/tools/pgindent/typedefs.list | 4 + 13 files changed, 873 insertions(+), 21 deletions(-) create mode 100644 src/test/subscription/t/027_worker_xact_stats.pl diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml index 5b10d18..22caeef 100644 --- a/doc/src/sgml/monitoring.sgml +++ b/doc/src/sgml/monitoring.sgml @@ -629,8 +629,8 @@ postgres 27093 0.0 0.0 30096 2752 ? Ss 11:34 0:00 postgres: ser <row> <entry><structname>pg_stat_subscription_workers</structname><indexterm><primary>pg_stat_subscription_workers</primary></indexterm></entry> - <entry>One row per subscription worker, showing statistics about errors - that occurred on that subscription worker. + <entry>One row per subscription worker, showing transaction statistics + and information about errors that occurred on that subscription worker. See <link linkend="monitoring-pg-stat-subscription-workers"> <structname>pg_stat_subscription_workers</structname></link> for details. </entry> @@ -3052,10 +3052,12 @@ SELECT pid, wait_event_type, wait_event FROM pg_stat_activity WHERE wait_event i <para> The <structname>pg_stat_subscription_workers</structname> view will contain - one row per subscription error reported by workers applying logical + one row per subscription, showing corresponding transaction statistics and + information about the error reported by workers applying logical replication changes and workers handling the initial data copy of the subscribed tables. The statistics entry is removed when the subscription - the worker is running on is removed. + the worker is running on is removed. The statistics of transaction size is + utilized only by the apply worker. </para> <table id="pg-stat-subscription-workers" xreflabel="pg_stat_subscription_workers"> @@ -3103,6 +3105,67 @@ SELECT pid, wait_event_type, wait_event FROM pg_stat_activity WHERE wait_event i <row> <entry role="catalog_table_entry"><para role="column_definition"> + <structfield>commit_count</structfield> <type>bigint</type> + </para> + <para> + Number of transactions successfully applied in this subscription. + COMMIT, COMMIT of streaming transaction and COMMIT PREPARED increments + this counter. + </para></entry> + </row> + + <row> + <entry role="catalog_table_entry"><para role="column_definition"> + <structfield>commit_bytes</structfield> <type>bigint</type> + </para> + <para> + Amount of data (in bytes) successfully applied in this subscription, + across <literal>commit_count</literal> transactions. + </para></entry> + </row> + + <row> + <entry role="catalog_table_entry"><para role="column_definition"> + <structfield>error_count</structfield> <type>bigint</type> + </para> + <para> + Number of transactions that failed to be applied by the table + sync worker or main apply worker in this subscription. + </para></entry> + </row> + + <row> + <entry role="catalog_table_entry"><para role="column_definition"> + <structfield>error_bytes</structfield> <type>bigint</type> + </para> + <para> + Amount of data (in bytes) unsuccessfully applied in this subscription, + across <literal>error_count</literal> transactions. + </para></entry> + </row> + + <row> + <entry role="catalog_table_entry"><para role="column_definition"> + <structfield>abort_count</structfield> <type>bigint</type> + </para> + <para> + Number of transactions aborted in this subscription. + ROLLBACK PREPARED increments this counter. + </para></entry> + </row> + + <row> + <entry role="catalog_table_entry"><para role="column_definition"> + <structfield>abort_bytes</structfield> <type>bigint</type> + </para> + <para> + Amount of data (in bytes) aborted in this subscription, + across <literal>abort_count</literal> transactions. + </para></entry> + </row> + + <row> + <entry role="catalog_table_entry"><para role="column_definition"> <structfield>last_error_relid</structfield> <type>oid</type> </para> <para> diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index ecf1a0b..a8880e8 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1267,6 +1267,12 @@ CREATE VIEW pg_stat_subscription_workers AS w.subid, s.subname, w.subrelid, + w.commit_count, + w.commit_bytes, + w.error_count, + w.error_bytes, + w.abort_count, + w.abort_bytes, w.last_error_relid, w.last_error_command, w.last_error_xid, diff --git a/src/backend/postmaster/pgstat.c b/src/backend/postmaster/pgstat.c index 209a2b4..5aa5e46 100644 --- a/src/backend/postmaster/pgstat.c +++ b/src/backend/postmaster/pgstat.c @@ -53,6 +53,7 @@ #include "postmaster/fork_process.h" #include "postmaster/interrupt.h" #include "postmaster/postmaster.h" +#include "replication/logicalworker.h" #include "replication/slot.h" #include "replication/walsender.h" #include "storage/backendid.h" @@ -325,11 +326,14 @@ static PgStat_StatTabEntry *pgstat_get_tab_entry(PgStat_StatDBEntry *dbentry, static PgStat_StatSubWorkerEntry *pgstat_get_subworker_entry(PgStat_StatDBEntry *dbentry, Oid subid, Oid subrelid, bool create); +static PgStat_SW_PreparedXactEntry *pgstat_get_subworker_prepared_txn(Oid databaseid, Oid subid, + char *gid, bool create); + static void pgstat_write_statsfiles(bool permanent, bool allDbs); static void pgstat_write_db_statsfile(PgStat_StatDBEntry *dbentry, bool permanent); static HTAB *pgstat_read_statsfiles(Oid onlydb, bool permanent, bool deep); static void pgstat_read_db_statsfile(Oid databaseid, HTAB *tabhash, HTAB *funchash, - HTAB *subworkerhash, bool permanent); + HTAB *subworkerhash, HTAB *preparedtxnhash, bool permanent); static void backend_read_statsfile(void); static bool pgstat_write_statsfile_needed(void); @@ -382,6 +386,9 @@ static void pgstat_recv_replslot(PgStat_MsgReplSlot *msg, int len); static void pgstat_recv_tempfile(PgStat_MsgTempFile *msg, int len); static void pgstat_recv_subscription_purge(PgStat_MsgSubscriptionPurge *msg, int len); static void pgstat_recv_subworker_error(PgStat_MsgSubWorkerError *msg, int len); +static void pgstat_recv_subworker_xact_end(PgStat_MsgSubWorkerXactEnd *msg, int len); +static void pgstat_recv_subworker_twophase_xact(PgStat_MsgSubWorkerTwophaseXact *msg, int len); + /* ------------------------------------------------------------ * Public functions called from postmaster follow @@ -1930,6 +1937,61 @@ pgstat_report_replslot_drop(const char *slotname) } /* ---------- + * pgstat_report_subworker_xact_end() - + * + * Tell the collector that worker transaction has successfully completed. + * ---------- + */ +void +pgstat_report_subworker_xact_end(Oid subid, Oid subrel, + LogicalRepMsgType command, PgStat_Counter xact_size) +{ + PgStat_MsgSubWorkerXactEnd msg; + + Assert(command == 0 /* table sync worker */ || + command == LOGICAL_REP_MSG_COMMIT || + command == LOGICAL_REP_MSG_STREAM_COMMIT); + + pgstat_setheader(&msg.m_hdr, PGSTAT_MTYPE_SUBWORKERXACTEND); + msg.m_databaseid = MyDatabaseId; + msg.m_subid = subid; + msg.m_subrelid = subrel; + msg.m_command = command; + msg.m_xact_bytes = xact_size; + pgstat_send(&msg, sizeof(PgStat_MsgSubWorkerXactEnd)); + + reset_apply_error_context_xact_size(); +} + +/* ---------- + * pgstat_report_subworker_twophase_xact() - + * + * Tell the collector that worker transaction has done 2PC related operation. + * ---------- + */ +void +pgstat_report_subworker_twophase_xact(Oid subid, LogicalRepMsgType command, + PgStat_Counter xact_size, char *gid) +{ + PgStat_MsgSubWorkerTwophaseXact msg; + + Assert(command == LOGICAL_REP_MSG_PREPARE || + command == LOGICAL_REP_MSG_STREAM_PREPARE || + command == LOGICAL_REP_MSG_COMMIT_PREPARED || + command == LOGICAL_REP_MSG_ROLLBACK_PREPARED); + + /* setup the message */ + pgstat_setheader(&msg.m_hdr, PGSTAT_MTYPE_SUBWORKERTWOPHASEXACT); + msg.m_databaseid = MyDatabaseId; + msg.m_subid = subid; + msg.m_command = command; + strlcpy(msg.m_gid, gid, sizeof(msg.m_gid)); + msg.m_xact_bytes = xact_size; + pgstat_send(&msg, sizeof(PgStat_MsgSubWorkerTwophaseXact)); + reset_apply_error_context_xact_size(); +} + +/* ---------- * pgstat_report_subworker_error() - * * Tell the collector about the subscription worker error. @@ -1950,6 +2012,7 @@ pgstat_report_subworker_error(Oid subid, Oid subrelid, Oid relid, msg.m_databaseid = MyDatabaseId; msg.m_subid = subid; msg.m_subrelid = subrelid; + msg.m_xact_error_bytes = get_apply_error_context_xact_size(); msg.m_relid = relid; msg.m_command = command; msg.m_xid = xid; @@ -1957,6 +2020,8 @@ pgstat_report_subworker_error(Oid subid, Oid subrelid, Oid relid, strlcpy(msg.m_message, errmsg, PGSTAT_SUBWORKERERROR_MSGLEN); pgstat_send(&msg, len); + + reset_apply_error_context_xact_size(); } /* ---------- @@ -3727,6 +3792,14 @@ PgstatCollectorMain(int argc, char *argv[]) pgstat_recv_subworker_error(&msg.msg_subworkererror, len); break; + case PGSTAT_MTYPE_SUBWORKERXACTEND: + pgstat_recv_subworker_xact_end(&msg.msg_subworkerxactend, len); + break; + + case PGSTAT_MTYPE_SUBWORKERTWOPHASEXACT: + pgstat_recv_subworker_twophase_xact(&msg.msg_subworkertwophasexact, len); + break; + default: break; } @@ -3833,6 +3906,14 @@ reset_dbentry_counters(PgStat_StatDBEntry *dbentry) PGSTAT_SUBWORKER_HASH_SIZE, &hash_ctl, HASH_ELEM | HASH_BLOBS); + + hash_ctl.keysize = sizeof(PgStat_SW_PreparedXactKey); + hash_ctl.entrysize = sizeof(PgStat_SW_PreparedXactEntry); + hash_ctl.hcxt = pgStatLocalContext; + dbentry->subworkers_preparedsizes = hash_create("Subscription worker stats of prepared txn", + PGSTAT_SUBWORKER_HASH_SIZE, + &hash_ctl, + HASH_ELEM | HASH_STRINGS | HASH_CONTEXT); } /* @@ -3943,6 +4024,12 @@ pgstat_get_subworker_entry(PgStat_StatDBEntry *dbentry, Oid subid, Oid subrelid, /* If not found, initialize the new one */ if (create && !found) { + subwentry->xact_commit_count = 0; + subwentry->xact_commit_bytes = 0; + subwentry->xact_error_count = 0; + subwentry->xact_error_bytes = 0; + subwentry->xact_abort_count = 0; + subwentry->xact_abort_bytes = 0; subwentry->relid = InvalidOid; subwentry->command = 0; subwentry->xid = InvalidTransactionId; @@ -4154,9 +4241,11 @@ pgstat_write_db_statsfile(PgStat_StatDBEntry *dbentry, bool permanent) HASH_SEQ_STATUS tstat; HASH_SEQ_STATUS fstat; HASH_SEQ_STATUS sstat; + HASH_SEQ_STATUS pstat; PgStat_StatTabEntry *tabentry; PgStat_StatFuncEntry *funcentry; PgStat_StatSubWorkerEntry *subwentry; + PgStat_SW_PreparedXactEntry *pentry; FILE *fpout; int32 format_id; Oid dbid = dbentry->databaseid; @@ -4223,6 +4312,17 @@ pgstat_write_db_statsfile(PgStat_StatDBEntry *dbentry, bool permanent) } /* + * Write subscription worker's prepared transaction struct + */ + hash_seq_init(&pstat, dbentry->subworkers_preparedsizes); + while ((pentry = (PgStat_SW_PreparedXactEntry *) hash_seq_search(&pstat)) != NULL) + { + fputc('P', fpout); + rc = fwrite(pentry, sizeof(PgStat_SW_PreparedXactEntry), 1, fpout); + (void) rc; /* we'll check for error with ferror */ + } + + /* * No more output to be done. Close the temp file and replace the old * pgstat.stat with it. The ferror() check replaces testing for error * after each individual fputc or fwrite above. @@ -4461,6 +4561,7 @@ pgstat_read_statsfiles(Oid onlydb, bool permanent, bool deep) dbentry->tables = NULL; dbentry->functions = NULL; dbentry->subworkers = NULL; + dbentry->subworkers_preparedsizes = NULL; /* * In the collector, disregard the timestamp we read from the @@ -4506,6 +4607,14 @@ pgstat_read_statsfiles(Oid onlydb, bool permanent, bool deep) &hash_ctl, HASH_ELEM | HASH_BLOBS | HASH_CONTEXT); + hash_ctl.keysize = sizeof(PgStat_SW_PreparedXactKey); + hash_ctl.entrysize = sizeof(PgStat_SW_PreparedXactEntry); + hash_ctl.hcxt = pgStatLocalContext; + dbentry->subworkers_preparedsizes = hash_create("Subscription worker stats of prepared txn", + PGSTAT_SUBWORKER_HASH_SIZE, + &hash_ctl, + HASH_ELEM | HASH_STRINGS | HASH_CONTEXT); + /* * If requested, read the data from the database-specific * file. Otherwise we just leave the hashtables empty. @@ -4515,6 +4624,7 @@ pgstat_read_statsfiles(Oid onlydb, bool permanent, bool deep) dbentry->tables, dbentry->functions, dbentry->subworkers, + dbentry->subworkers_preparedsizes, permanent); break; @@ -4599,7 +4709,7 @@ done: */ static void pgstat_read_db_statsfile(Oid databaseid, HTAB *tabhash, HTAB *funchash, - HTAB *subworkerhash, bool permanent) + HTAB *subworkerhash, HTAB *preparedtxnhash, bool permanent) { PgStat_StatTabEntry *tabentry; PgStat_StatTabEntry tabbuf; @@ -4755,6 +4865,32 @@ pgstat_read_db_statsfile(Oid databaseid, HTAB *tabhash, HTAB *funchash, memcpy(subwentry, &subwbuf, sizeof(subwbuf)); break; + case 'P': + { + PgStat_SW_PreparedXactEntry buff; + PgStat_SW_PreparedXactEntry *prepared_xact_entry; + + if (fread(&buff, 1, sizeof(PgStat_SW_PreparedXactEntry), + fpin) != sizeof(PgStat_SW_PreparedXactEntry)) + { + ereport(pgStatRunningInCollector ? LOG : WARNING, + (errmsg("corrupted statistics file \"%s\"", + statfile))); + goto done; + } + + if (preparedtxnhash == NULL) + break; + + prepared_xact_entry = + (PgStat_SW_PreparedXactEntry *) hash_search(preparedtxnhash, + (void *) &buff.key, + HASH_ENTER, NULL); + + memcpy(prepared_xact_entry, &buff, sizeof(PgStat_SW_PreparedXactEntry)); + break; + } + /* * 'E' The EOF marker of a complete stats file. */ @@ -5430,6 +5566,8 @@ pgstat_recv_dropdb(PgStat_MsgDropdb *msg, int len) hash_destroy(dbentry->functions); if (dbentry->subworkers != NULL) hash_destroy(dbentry->subworkers); + if (dbentry->subworkers_preparedsizes != NULL) + hash_destroy(dbentry->subworkers_preparedsizes); if (hash_search(pgStatDBHash, (void *) &dbid, @@ -5469,10 +5607,13 @@ pgstat_recv_resetcounter(PgStat_MsgResetcounter *msg, int len) hash_destroy(dbentry->functions); if (dbentry->subworkers != NULL) hash_destroy(dbentry->subworkers); + if (dbentry->subworkers_preparedsizes != NULL) + hash_destroy(dbentry->subworkers_preparedsizes); dbentry->tables = NULL; dbentry->functions = NULL; dbentry->subworkers = NULL; + dbentry->subworkers_preparedsizes = NULL; /* * Reset database-level stats, too. This creates empty hash tables for @@ -5548,10 +5689,28 @@ pgstat_recv_resetsinglecounter(PgStat_MsgResetsinglecounter *msg, int len) else if (msg->m_resettype == RESET_SUBWORKER) { PgStat_StatSubWorkerKey key; + PgStat_SW_PreparedXactKey pkey; key.subid = msg->m_objectid; key.subrelid = msg->m_subobjectid; (void) hash_search(dbentry->subworkers, (void *) &key, HASH_REMOVE, NULL); + + /* + * Clean up entry of prepared size hash as well. + * + * There's no gid input here as a part of key for + * PgStat_SW_PreparedXactEntry. Also any valid 'relid' indicates the + * entry is created by table sync but it has nothing to do with the + * two phase operation of apply. Proceed with this removal only when + * the specified subrelid isn't invalid. + */ + if (msg->m_subobjectid == InvalidOid) + { + pkey.subid = msg->m_objectid; + (void) hash_search(dbentry->subworkers_preparedsizes, + (void *) &pkey, HASH_REMOVE, NULL); + } + } } @@ -6107,6 +6266,7 @@ pgstat_recv_subscription_purge(PgStat_MsgSubscriptionPurge *msg, int len) HASH_SEQ_STATUS hstat; PgStat_StatDBEntry *dbentry; PgStat_StatSubWorkerEntry *subwentry; + PgStat_SW_PreparedXactEntry *prepared_txn_entry; dbentry = pgstat_get_db_entry(msg->m_databaseid, false); @@ -6128,6 +6288,127 @@ pgstat_recv_subscription_purge(PgStat_MsgSubscriptionPurge *msg, int len) } } } + + /* Remove associated prepared transaction stats */ + if (!dbentry->subworkers_preparedsizes) + { + hash_seq_init(&hstat, dbentry->subworkers_preparedsizes); + while ((prepared_txn_entry = (PgStat_SW_PreparedXactEntry *) hash_seq_search(&hstat)) != NULL) + { + for (int i = 0; i < msg->m_nentries; i++) + { + if (prepared_txn_entry->key.subid == msg->m_subids[i]) + (void) hash_search(dbentry->subworkers_preparedsizes, + (void *) &(prepared_txn_entry->key), + HASH_REMOVE, NULL); + break; + } + } + } +} + +/* ---------- + * pgstat_recv_subworker_xact_end() - + * + * Process a SUBWORKERXACTEND message. + * ---------- + */ +static void +pgstat_recv_subworker_xact_end(PgStat_MsgSubWorkerXactEnd *msg, int len) +{ + PgStat_StatDBEntry *dbentry; + PgStat_StatSubWorkerEntry *wentry; + + dbentry = pgstat_get_db_entry(msg->m_databaseid, true); + wentry = pgstat_get_subworker_entry(dbentry, msg->m_subid, + msg->m_subrelid, true); + Assert(wentry); + + /* table sync worker */ + if (msg->m_command == 0) + wentry->xact_commit_count++; + else + { + /* apply worker */ + switch (msg->m_command) + { + case LOGICAL_REP_MSG_COMMIT: + case LOGICAL_REP_MSG_STREAM_COMMIT: + wentry->xact_commit_count++; + wentry->xact_commit_bytes += msg->m_xact_bytes; + break; + default: + elog(ERROR, "unexpected logical message type as normal apply end"); + break; + } + } +} + +/* ---------- + * pgstat_recv_subworker_twophase_xact() - + * + * Process a SUBWORKERTWOPHASEXACT message. + * ---------- + */ +static void +pgstat_recv_subworker_twophase_xact(PgStat_MsgSubWorkerTwophaseXact *msg, int len) +{ + PgStat_StatDBEntry *dbentry; + PgStat_SW_PreparedXactEntry *pentry; + PgStat_StatSubWorkerEntry *wentry; + PgStat_SW_PreparedXactKey key; + + pentry = pgstat_get_subworker_prepared_txn(msg->m_databaseid, + msg->m_subid, + msg->m_gid, true); + Assert(pentry); + switch (msg->m_command) + { + case LOGICAL_REP_MSG_PREPARE: + case LOGICAL_REP_MSG_STREAM_PREPARE: + + /* + * Make each size of prepared transaction persistent so that we + * can update stats over the server restart and make prepared + * stats updated when commit prepared or rollback prepared + * arrives. + */ + pentry->subid = msg->m_subid; + strlcpy(pentry->gid, msg->m_gid, sizeof(pentry->gid)); + pentry->xact_size = msg->m_xact_bytes; + break; + + case LOGICAL_REP_MSG_COMMIT_PREPARED: + case LOGICAL_REP_MSG_ROLLBACK_PREPARED: + /* Update exported xact stats now */ + dbentry = pgstat_get_db_entry(msg->m_databaseid, true); + wentry = pgstat_get_subworker_entry(dbentry, + msg->m_subid, + InvalidOid /* apply worker */ , + true); + Assert(wentry); + if (msg->m_command == LOGICAL_REP_MSG_COMMIT_PREPARED) + { + wentry->xact_commit_count++; + wentry->xact_commit_bytes += pentry->xact_size; + } + else + { + wentry->xact_abort_count++; + wentry->xact_abort_bytes += pentry->xact_size; + } + + /* Clean up this gid from transaction size hash */ + key.subid = pentry->subid; + strlcpy(key.gid, msg->m_gid, sizeof(key.gid)); + (void) hash_search(dbentry->subworkers_preparedsizes, + (void *) &key, HASH_REMOVE, NULL); + break; + + default: + elog(ERROR, "unexpected logical message type as prepare transaction"); + break; + } } /* ---------- @@ -6149,6 +6430,10 @@ pgstat_recv_subworker_error(PgStat_MsgSubWorkerError *msg, int len) msg->m_subrelid, true); Assert(subwentry); + /* general transaction stats for error */ + subwentry->xact_error_count++; + subwentry->xact_error_bytes += msg->m_xact_error_bytes; + /* * Update only the counter and last error timestamp if we received * the same error again @@ -6396,3 +6681,38 @@ pgstat_count_slru_truncate(int slru_idx) { slru_entry(slru_idx)->m_truncate += 1; } + + /* ---------- + * pgstat_get_subworker_prepared_txn + * + * Return subscription worker entry with the given subscription OID and + * gid. + * ---------- + */ +static PgStat_SW_PreparedXactEntry * +pgstat_get_subworker_prepared_txn(Oid databaseid, Oid subid, + char *gid, bool create) +{ + PgStat_StatDBEntry *dbentry; + PgStat_SW_PreparedXactKey key; + PgStat_SW_PreparedXactEntry *pentry; + HASHACTION action; + bool found; + + dbentry = pgstat_get_db_entry(databaseid, true); + key.subid = subid; + strlcpy(key.gid, gid, sizeof(key.gid)); + action = (create ? HASH_ENTER : HASH_FIND); + pentry = (PgStat_SW_PreparedXactEntry *) hash_search(dbentry->subworkers_preparedsizes, + (void *) &key, + action, &found); + + if (create && !found) + { + pentry->subid = 0; + pentry->gid[0] = '\0'; + pentry->xact_size = 0; + } + + return pentry; +} diff --git a/src/backend/replication/logical/proto.c b/src/backend/replication/logical/proto.c index 9f5bf4b..3cd933a 100644 --- a/src/backend/replication/logical/proto.c +++ b/src/backend/replication/logical/proto.c @@ -17,6 +17,7 @@ #include "catalog/pg_type.h" #include "libpq/pqformat.h" #include "replication/logicalproto.h" +#include "replication/logicalworker.h" #include "utils/lsyscache.h" #include "utils/syscache.h" @@ -842,6 +843,7 @@ logicalrep_read_tuple(StringInfo in, LogicalRepTupleData *tuple) { int i; int natts; + Size total_size; /* Get number of attributes */ natts = pq_getmsgint(in, 2); @@ -850,6 +852,7 @@ logicalrep_read_tuple(StringInfo in, LogicalRepTupleData *tuple) tuple->colvalues = (StringInfoData *) palloc0(natts * sizeof(StringInfoData)); tuple->colstatus = (char *) palloc(natts * sizeof(char)); tuple->ncols = natts; + total_size = natts * (sizeof(char) + sizeof(StringInfoData)); /* Read the data */ for (i = 0; i < natts; i++) @@ -880,6 +883,9 @@ logicalrep_read_tuple(StringInfo in, LogicalRepTupleData *tuple) value->len = len; value->cursor = 0; value->maxlen = len; + + /* memory for tuple */ + total_size += len + 1; break; case LOGICALREP_COLUMN_BINARY: len = pq_getmsgint(in, 4); /* read length */ @@ -893,11 +899,17 @@ logicalrep_read_tuple(StringInfo in, LogicalRepTupleData *tuple) value->len = len; value->cursor = 0; value->maxlen = len; + + /* memory for tuple */ + total_size += len + 1; break; default: elog(ERROR, "unrecognized data representation type '%c'", kind); } } + + /* Record memory consumption for tuple */ + add_apply_error_context_xact_size(total_size); } /* diff --git a/src/backend/replication/logical/tablesync.c b/src/backend/replication/logical/tablesync.c index f07983a..5bb1d63 100644 --- a/src/backend/replication/logical/tablesync.c +++ b/src/backend/replication/logical/tablesync.c @@ -1149,6 +1149,12 @@ copy_table_done: MyLogicalRepWorker->relstate_lsn = *origin_startpos; SpinLockRelease(&MyLogicalRepWorker->relmutex); + /* Report the success of table sync. */ + pgstat_report_subworker_xact_end(MyLogicalRepWorker->subid, + MyLogicalRepWorker->relid, + 0 /* no logical message type */ , + 0 /* xact size */ ); + /* * Finally, wait until the main apply worker tells us to catch up and then * return to let LogicalRepApplyLoop do it. diff --git a/src/backend/replication/logical/worker.c b/src/backend/replication/logical/worker.c index 2e79302..6617437 100644 --- a/src/backend/replication/logical/worker.c +++ b/src/backend/replication/logical/worker.c @@ -221,6 +221,18 @@ typedef struct ApplyErrorCallbackArg LogicalRepMsgType command; /* 0 if invalid */ LogicalRepRelMapEntry *rel; + /* + * Store data size of this transaction. + * + * The byte size of transaction on the publisher is calculated by + * ReorderBufferChangeSize() based on the ReorderBufferChange structure. + * But on the subscriber, consumed resources are not same as the + * publisher's decoding processsing and required to be computed in + * different way. Therefore, the exact same byte size is not restored on + * the subscriber usually. + */ + PgStat_Counter bytes; + /* Remote node information */ int remote_attnum; /* -1 if invalid */ TransactionId remote_xid; @@ -231,6 +243,7 @@ static ApplyErrorCallbackArg apply_error_callback_arg = { .command = 0, .rel = NULL, + .bytes = 0, .remote_attnum = -1, .remote_xid = InvalidTransactionId, .ts = 0, @@ -293,6 +306,7 @@ static inline void cleanup_subxact_info(void); static void stream_cleanup_files(Oid subid, TransactionId xid); static void stream_open_file(Oid subid, TransactionId xid, bool first); static void stream_write_change(char action, StringInfo s); + static void stream_close_file(void); static void send_feedback(XLogRecPtr recvpos, bool force, bool requestReply); @@ -304,6 +318,9 @@ static void maybe_reread_subscription(void); /* prototype needed because of stream_commit */ static void apply_dispatch(StringInfo s); +static void update_apply_change_size(LogicalRepMsgType action, + bool is_partitioned); + static void apply_handle_commit_internal(LogicalRepCommitData *commit_data); static void apply_handle_insert_internal(ApplyExecutionData *edata, ResultRelInfo *relinfo, @@ -818,6 +835,11 @@ apply_handle_commit(StringInfo s) /* Process any tables that are being synchronized in parallel. */ process_syncing_tables(commit_data.end_lsn); + pgstat_report_subworker_xact_end(MyLogicalRepWorker->subid, + InvalidOid, + LOGICAL_REP_MSG_COMMIT, + get_apply_error_context_xact_size()); + pgstat_report_activity(STATE_IDLE, NULL); reset_apply_error_context_info(); } @@ -922,6 +944,13 @@ apply_handle_prepare(StringInfo s) /* Process any tables that are being synchronized in parallel. */ process_syncing_tables(prepare_data.end_lsn); + /* Update the memory consumption of prepare and report */ + update_apply_change_size(LOGICAL_REP_MSG_PREPARE, false); + pgstat_report_subworker_twophase_xact(MyLogicalRepWorker->subid, + LOGICAL_REP_MSG_PREPARE, + get_apply_error_context_xact_size(), + prepare_data.gid); + pgstat_report_activity(STATE_IDLE, NULL); reset_apply_error_context_info(); } @@ -963,6 +992,13 @@ apply_handle_commit_prepared(StringInfo s) /* Process any tables that are being synchronized in parallel. */ process_syncing_tables(prepare_data.end_lsn); + /* Update the transaction size and report */ + update_apply_change_size(LOGICAL_REP_MSG_COMMIT_PREPARED, false); + pgstat_report_subworker_twophase_xact(MyLogicalRepWorker->subid, + LOGICAL_REP_MSG_COMMIT_PREPARED, + get_apply_error_context_xact_size(), + prepare_data.gid); + pgstat_report_activity(STATE_IDLE, NULL); reset_apply_error_context_info(); } @@ -1014,6 +1050,13 @@ apply_handle_rollback_prepared(StringInfo s) /* Process any tables that are being synchronized in parallel. */ process_syncing_tables(rollback_data.rollback_end_lsn); + /* Update the transaction size and report */ + update_apply_change_size(LOGICAL_REP_MSG_ROLLBACK_PREPARED, false); + pgstat_report_subworker_twophase_xact(MyLogicalRepWorker->subid, + LOGICAL_REP_MSG_ROLLBACK_PREPARED, + get_apply_error_context_xact_size(), + rollback_data.gid); + pgstat_report_activity(STATE_IDLE, NULL); reset_apply_error_context_info(); } @@ -1066,6 +1109,15 @@ apply_handle_stream_prepare(StringInfo s) /* Process any tables that are being synchronized in parallel. */ process_syncing_tables(prepare_data.end_lsn); + /* + * Report the prepared streaming xact size to the stats collector in a + * prepared xact manner to make it survive over the restart. + */ + pgstat_report_subworker_twophase_xact(MyLogicalRepWorker->subid, + LOGICAL_REP_MSG_STREAM_PREPARE, + get_apply_error_context_xact_size(), + prepare_data.gid); + pgstat_report_activity(STATE_IDLE, NULL); reset_apply_error_context_info(); @@ -1438,6 +1490,12 @@ apply_handle_stream_commit(StringInfo s) /* Process any tables that are being synchronized in parallel. */ process_syncing_tables(commit_data.end_lsn); + /* Report and clean up the xid */ + pgstat_report_subworker_xact_end(MyLogicalRepWorker->subid, + InvalidOid, + LOGICAL_REP_MSG_STREAM_COMMIT, + get_apply_error_context_xact_size()); + pgstat_report_activity(STATE_IDLE, NULL); reset_apply_error_context_info(); @@ -1579,6 +1637,10 @@ apply_handle_insert(StringInfo s) slot_fill_defaults(rel, estate, remoteslot); MemoryContextSwitchTo(oldctx); + /* Update transaction size */ + update_apply_change_size(LOGICAL_REP_MSG_INSERT, + rel->localrel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE); + /* For a partitioned table, insert the tuple into a partition. */ if (rel->localrel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE) apply_handle_tuple_routing(edata, @@ -1736,7 +1798,11 @@ apply_handle_update(StringInfo s) has_oldtup ? &oldtup : &newtup); MemoryContextSwitchTo(oldctx); - /* For a partitioned table, apply update to correct partition. */ + /* Update transaction size */ + update_apply_change_size(LOGICAL_REP_MSG_UPDATE, + rel->localrel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE); + + /* For a partitioned table, apply update to corect partition. */ if (rel->localrel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE) apply_handle_tuple_routing(edata, remoteslot, &newtup, CMD_UPDATE); @@ -1870,6 +1936,10 @@ apply_handle_delete(StringInfo s) slot_store_data(remoteslot, rel, &oldtup); MemoryContextSwitchTo(oldctx); + /* Update transaction size */ + update_apply_change_size(LOGICAL_REP_MSG_DELETE, + rel->localrel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE); + /* For a partitioned table, apply delete to correct partition. */ if (rel->localrel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE) apply_handle_tuple_routing(edata, @@ -2421,6 +2491,81 @@ apply_dispatch(StringInfo s) } /* + * Subscriber side implementation equivalent to ReorderBufferChangeSize + * of the publisher. + * + * According to the logical replication message type, record major + * resource consumptions of this subscription for each message. + * At present, do not collect data from generic functions to keep + * code simplicity except for tuple length, since the implementation + * complexity versus benefit tradeoff should not be good. + * In terms of tuple length consumption, see logicalrep_read_tuple. + * + * Also, add multiple values at once in order to reduce the number + * of calls to this function, although the disadvantage of this way + * is we cannot get correct transaction size when we get an error. + * + * 'is_partitioned' is used to add some extra size. + */ +static void +update_apply_change_size(LogicalRepMsgType action, bool is_partitioned) +{ + int64 size = 0; + + switch (action) + { + /* No special memory consumption */ + case LOGICAL_REP_MSG_BEGIN: + case LOGICAL_REP_MSG_COMMIT: + case LOGICAL_REP_MSG_TRUNCATE: + case LOGICAL_REP_MSG_TYPE: + case LOGICAL_REP_MSG_ORIGIN: + case LOGICAL_REP_MSG_MESSAGE: + case LOGICAL_REP_MSG_BEGIN_PREPARE: + case LOGICAL_REP_MSG_RELATION: + break; + + case LOGICAL_REP_MSG_INSERT: + case LOGICAL_REP_MSG_UPDATE: + case LOGICAL_REP_MSG_DELETE: + Assert(!in_streamed_transaction); + + /* + * Compute size based on ApplyExecutionData. The size of + * LogicalRepRelMapEntry can be skipped because it is obtained + * from hash_search in logicalrep_rel_open. + */ + size += sizeof(ApplyExecutionData) + sizeof(EState) + + sizeof(ResultRelInfo) + sizeof(ResultRelInfo); + + /* Add some extra size if the target relation is partitioned. */ + if (is_partitioned) + size += sizeof(ModifyTableState) + sizeof(PartitionTupleRouting); + break; + + case LOGICAL_REP_MSG_PREPARE: + case LOGICAL_REP_MSG_COMMIT_PREPARED: + case LOGICAL_REP_MSG_ROLLBACK_PREPARED: + size += sizeof(FlushPosition); + break; + + case LOGICAL_REP_MSG_STREAM_START: + case LOGICAL_REP_MSG_STREAM_STOP: + case LOGICAL_REP_MSG_STREAM_COMMIT: + case LOGICAL_REP_MSG_STREAM_PREPARE: + case LOGICAL_REP_MSG_STREAM_ABORT: + default: + ereport(ERROR, + (errcode(ERRCODE_PROTOCOL_VIOLATION), + errmsg("invalid logical replication message type \"%c\"", action))); + } + + /* Update the total size of consumption when necessary */ + if (size != 0) + add_apply_error_context_xact_size(size); +} + +/* * Figure out which write/flush positions to report to the walsender process. * * We can't simply report back the last LSN the walsender sent us because the @@ -3294,6 +3439,7 @@ stream_write_change(char action, StringInfo s) BufFileWrite(stream_fd, &s->data[s->cursor], len); } + /* * Cleanup the memory for subxacts and reset the related variables. */ @@ -3604,6 +3750,27 @@ ApplyWorkerMain(Datum main_arg) proc_exit(0); } +/* Exported so that stats collector can utilize this value */ +int64 +get_apply_error_context_xact_size(void) +{ + return apply_error_callback_arg.bytes; +} + +/* Add size to apply error bytes */ +void +add_apply_error_context_xact_size(int64 size) +{ + apply_error_callback_arg.bytes += size; +} + +/* Reset information of apply error callback */ +void +reset_apply_error_context_xact_size(void) +{ + apply_error_callback_arg.bytes = 0; +} + /* * Is current process a logical replication worker? */ diff --git a/src/backend/utils/adt/pgstatfuncs.c b/src/backend/utils/adt/pgstatfuncs.c index a34c0b6..d7264fc 100644 --- a/src/backend/utils/adt/pgstatfuncs.c +++ b/src/backend/utils/adt/pgstatfuncs.c @@ -2414,7 +2414,7 @@ pg_stat_get_replication_slot(PG_FUNCTION_ARGS) Datum pg_stat_get_subscription_worker(PG_FUNCTION_ARGS) { -#define PG_STAT_GET_SUBSCRIPTION_WORKER_COLS 9 +#define PG_STAT_GET_SUBSCRIPTION_WORKER_COLS 15 Oid subid = PG_GETARG_OID(0); Oid subrelid; TupleDesc tupdesc; @@ -2441,19 +2441,31 @@ pg_stat_get_subscription_worker(PG_FUNCTION_ARGS) OIDOID, -1, 0); TupleDescInitEntry(tupdesc, (AttrNumber) 2, "subrelid", OIDOID, -1, 0); - TupleDescInitEntry(tupdesc, (AttrNumber) 3, "last_error_relid", + TupleDescInitEntry(tupdesc, (AttrNumber) 3, "commit_count", + INT8OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) 4, "commit_bytes", + INT8OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) 5, "error_count", + INT8OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) 6, "error_bytes", + INT8OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) 7, "abort_count", + INT8OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) 8, "abort_bytes", + INT8OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) 9, "last_error_relid", OIDOID, -1, 0); - TupleDescInitEntry(tupdesc, (AttrNumber) 4, "last_error_command", + TupleDescInitEntry(tupdesc, (AttrNumber) 10, "last_error_command", TEXTOID, -1, 0); - TupleDescInitEntry(tupdesc, (AttrNumber) 5, "last_error_xid", + TupleDescInitEntry(tupdesc, (AttrNumber) 11, "last_error_xid", XIDOID, -1, 0); - TupleDescInitEntry(tupdesc, (AttrNumber) 6, "last_error_count", + TupleDescInitEntry(tupdesc, (AttrNumber) 12, "last_error_count", INT8OID, -1, 0); - TupleDescInitEntry(tupdesc, (AttrNumber) 7, "last_error_message", + TupleDescInitEntry(tupdesc, (AttrNumber) 13, "last_error_message", TEXTOID, -1, 0); - TupleDescInitEntry(tupdesc, (AttrNumber) 8, "first_error_time", + TupleDescInitEntry(tupdesc, (AttrNumber) 14, "first_error_time", TIMESTAMPTZOID, -1, 0); - TupleDescInitEntry(tupdesc, (AttrNumber) 9, "last_error_time", + TupleDescInitEntry(tupdesc, (AttrNumber) 15, "last_error_time", TIMESTAMPTZOID, -1, 0); BlessTupleDesc(tupdesc); @@ -2471,6 +2483,14 @@ pg_stat_get_subscription_worker(PG_FUNCTION_ARGS) else nulls[i++] = true; + /* transaction stats */ + values[i++] = Int64GetDatum(wentry->xact_commit_count); + values[i++] = Int64GetDatum(wentry->xact_commit_bytes); + values[i++] = Int64GetDatum(wentry->xact_error_count); + values[i++] = Int64GetDatum(wentry->xact_error_bytes); + values[i++] = Int64GetDatum(wentry->xact_abort_count); + values[i++] = Int64GetDatum(wentry->xact_abort_bytes); + /* last_error_relid */ if (OidIsValid(wentry->relid)) values[i++] = ObjectIdGetDatum(wentry->relid); diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index 25f685f..fc654b4 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -5389,9 +5389,9 @@ proname => 'pg_stat_get_subscription_worker', prorows => '1', proisstrict => 'f', proretset => 't', provolatile => 's', proparallel => 'r', prorettype => 'record', proargtypes => 'oid oid', - proallargtypes => '{oid,oid,oid,oid,oid,text,xid,int8,text,timestamptz,timestamptz}', - proargmodes => '{i,i,o,o,o,o,o,o,o,o,o}', - proargnames => '{subid,subrelid,subid,subrelid,last_error_relid,last_error_command,last_error_xid,last_error_count,last_error_message,first_error_time,last_error_time}', + proallargtypes => '{oid,oid,oid,oid,int8,int8,int8,int8,int8,int8,oid,text,xid,int8,text,timestamptz,timestamptz}', + proargmodes => '{i,i,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o}', + proargnames => '{subid,subrelid,subid,subrelid,commit_count,commit_bytes,error_count,error_bytes,abort_count,abort_bytes,last_error_relid,last_error_command,last_error_xid,last_error_count,last_error_message,first_error_time,last_error_time}', prosrc => 'pg_stat_get_subscription_worker' }, { oid => '6118', descr => 'statistics: information about subscription', proname => 'pg_stat_get_subscription', prorows => '10', proisstrict => 'f', diff --git a/src/include/pgstat.h b/src/include/pgstat.h index 2c26b1c..266711b 100644 --- a/src/include/pgstat.h +++ b/src/include/pgstat.h @@ -86,6 +86,8 @@ typedef enum StatMsgType PGSTAT_MTYPE_DISCONNECT, PGSTAT_MTYPE_SUBSCRIPTIONPURGE, PGSTAT_MTYPE_SUBWORKERERROR, + PGSTAT_MTYPE_SUBWORKERXACTEND, + PGSTAT_MTYPE_SUBWORKERTWOPHASEXACT, } StatMsgType; /* ---------- @@ -558,6 +560,56 @@ typedef struct PgStat_MsgSubscriptionPurge } PgStat_MsgSubscriptionPurge; /* ---------- + * PgStat_MsgSubscriptionXactEnd Sent by the apply worker or the table sync worker + * to report successful transaction ends. + * ---------- + */ +typedef struct PgStat_MsgSubWorkerXactEnd +{ + PgStat_MsgHdr m_hdr; + + /* determine the worker entry */ + Oid m_databaseid; + Oid m_subid; + Oid m_subrelid; + + /* + * distinguish between transaction commits and streaming transaction + * aborts that are handled without error. + */ + LogicalRepMsgType m_command; + + /* memory consumption used by transaction */ + PgStat_Counter m_xact_bytes; + +} PgStat_MsgSubWorkerXactEnd; + +/* ---------- + * PgStat_MsgSubWorkerTwophaseXact Sent by the apply worker to make size of prepared + * txn persistent over the server restart and make it + * visible after commit prepare or rollback prepared. + * This is separated from PgStat_MsgSubWorkerXactEnd + * so that we can reduce message size of gid for other + * operations (e.g. normal COMMIT) that should happen more + * frequently than prepare operation usually. + * ---------- + */ +typedef struct PgStat_MsgSubWorkerTwophaseXact +{ + PgStat_MsgHdr m_hdr; + + /* determine the subscription */ + Oid m_databaseid; + Oid m_subid; + + LogicalRepMsgType m_command; + char m_gid[GIDSIZE]; + int gid_len; + PgStat_Counter m_xact_bytes; + +} PgStat_MsgSubWorkerTwophaseXact; + +/* ---------- * PgStat_MsgSubWorkerError Sent by the apply worker or the table sync worker to * report the error occurred during logical replication. * ---------- @@ -577,6 +629,12 @@ typedef struct PgStat_MsgSubWorkerError Oid m_subrelid; /* + * Transaction stats of subscription needs to be updated when an error + * occurs. + */ + PgStat_Counter m_xact_error_bytes; + + /* * Oid of the table that the reporter was actually processing. m_relid can * be InvalidOid if an error occurred during worker applying a * non-data-modification message such as RELATION. @@ -768,6 +826,8 @@ typedef union PgStat_Msg PgStat_MsgDisconnect msg_disconnect; PgStat_MsgSubscriptionPurge msg_subscriptionpurge; PgStat_MsgSubWorkerError msg_subworkererror; + PgStat_MsgSubWorkerXactEnd msg_subworkerxactend; + PgStat_MsgSubWorkerTwophaseXact msg_subworkertwophasexact; } PgStat_Msg; @@ -822,16 +882,23 @@ typedef struct PgStat_StatDBEntry TimestampTz stats_timestamp; /* time of db stats file update */ /* - * tables, functions, and subscription workers must be last in the struct, - * because we don't write the pointers out to the stats file. + * tables, functions, subscription workers and its prepared transaction + * stats must be last in the struct, because we don't write the pointers + * out to the stats file. * * subworker is the hash table of PgStat_StatSubWorkerEntry which stores * statistics of logical replication workers: apply worker and table sync * worker. + * + * subworkers_preparedsizes should give appropriate transaction sizes at + * either commit prepared or rollback prepared time, even when it's after + * the server restart. We have the apply worker send those statistics to + * the stats collector at prepare time. */ HTAB *tables; HTAB *functions; HTAB *subworkers; + HTAB *subworkers_preparedsizes; } PgStat_StatDBEntry; @@ -1005,6 +1072,16 @@ typedef struct PgStat_StatSubWorkerEntry PgStat_StatSubWorkerKey key; /* hash key (must be first) */ /* + * Cumulative transaction statistics of subscription worker + */ + PgStat_Counter xact_commit_count; + PgStat_Counter xact_commit_bytes; + PgStat_Counter xact_error_count; + PgStat_Counter xact_error_bytes; + PgStat_Counter xact_abort_count; + PgStat_Counter xact_abort_bytes; + + /* * Subscription worker error statistics representing an error that * occurred during application of logical replication or the initial table * synchronization. @@ -1018,6 +1095,22 @@ typedef struct PgStat_StatSubWorkerEntry char error_message[PGSTAT_SUBWORKERERROR_MSGLEN]; } PgStat_StatSubWorkerEntry; +/* prepared transaction */ +typedef struct PgStat_SW_PreparedXactKey +{ + Oid subid; + char gid[GIDSIZE]; +} PgStat_SW_PreparedXactKey; + +typedef struct PgStat_SW_PreparedXactEntry +{ + PgStat_SW_PreparedXactKey key; /* hash key */ + + Oid subid; + char gid[GIDSIZE]; + PgStat_Counter xact_size; +} PgStat_SW_PreparedXactEntry; + /* * Working state needed to accumulate per-function-call timing statistics. */ @@ -1128,6 +1221,10 @@ extern void pgstat_report_checksum_failure(void); extern void pgstat_report_replslot(const PgStat_StatReplSlotEntry *repSlotStat); extern void pgstat_report_replslot_create(const char *slotname); extern void pgstat_report_replslot_drop(const char *slotname); +extern void pgstat_report_subworker_xact_end(Oid subid, Oid subrel, + LogicalRepMsgType command, PgStat_Counter xact_size); +extern void pgstat_report_subworker_twophase_xact(Oid subid, LogicalRepMsgType command, + PgStat_Counter xact_size, char *gid); extern void pgstat_report_subworker_error(Oid subid, Oid subrelid, Oid relid, LogicalRepMsgType command, TransactionId xid, const char *errmsg); diff --git a/src/include/replication/logicalworker.h b/src/include/replication/logicalworker.h index 2ad61a0..9a8447b 100644 --- a/src/include/replication/logicalworker.h +++ b/src/include/replication/logicalworker.h @@ -16,4 +16,9 @@ extern void ApplyWorkerMain(Datum main_arg); extern bool IsLogicalWorker(void); +/* for transaction stats */ +extern int64 get_apply_error_context_xact_size(void); +extern void add_apply_error_context_xact_size(int64 size); +extern void reset_apply_error_context_xact_size(void); + #endif /* LOGICALWORKER_H */ diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out index d60c5a5..bd3e221 100644 --- a/src/test/regress/expected/rules.out +++ b/src/test/regress/expected/rules.out @@ -2097,6 +2097,12 @@ pg_stat_subscription| SELECT su.oid AS subid, pg_stat_subscription_workers| SELECT w.subid, s.subname, w.subrelid, + w.commit_count, + w.commit_bytes, + w.error_count, + w.error_bytes, + w.abort_count, + w.abort_bytes, w.last_error_relid, w.last_error_command, w.last_error_xid, @@ -2112,7 +2118,7 @@ pg_stat_subscription_workers| SELECT w.subid, pg_subscription_rel.srrelid AS relid FROM pg_subscription_rel WHERE (pg_subscription_rel.srsubstate <> 'r'::"char")) sr, - (LATERAL pg_stat_get_subscription_worker(sr.subid, sr.relid) w(subid, subrelid, last_error_relid, last_error_command, last_error_xid, last_error_count, last_error_message, first_error_time, last_error_time) + (LATERAL pg_stat_get_subscription_worker(sr.subid, sr.relid) w(subid, subrelid, commit_count, commit_bytes, error_count, error_bytes, abort_count, abort_bytes, last_error_relid, last_error_command, last_error_xid, last_error_count, last_error_message, first_error_time, last_error_time) JOIN pg_subscription s ON ((w.subid = s.oid))); pg_stat_sys_indexes| SELECT pg_stat_all_indexes.relid, pg_stat_all_indexes.indexrelid, diff --git a/src/test/subscription/t/027_worker_xact_stats.pl b/src/test/subscription/t/027_worker_xact_stats.pl new file mode 100644 index 0000000..e09be42 --- /dev/null +++ b/src/test/subscription/t/027_worker_xact_stats.pl @@ -0,0 +1,146 @@ + +# Copyright (c) 2021, PostgreSQL Global Development Group + +# Tests for subscription worker statistics during apply. +use strict; +use warnings; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More tests => 2; + +# Create publisher node +my $node_publisher = PostgreSQL::Test::Cluster->new('publisher'); +$node_publisher->init(allows_streaming => 'logical'); +$node_publisher->append_conf( + 'postgresql.conf', qq[ +logical_decoding_work_mem = 64kB +max_prepared_transactions = 10 +max_wal_senders = 10 +wal_sender_timeout = 0 +]); +$node_publisher->start; + +# Create subscriber node +my $node_subscriber = PostgreSQL::Test::Cluster->new('subscriber'); +$node_subscriber->init(allows_streaming => 'logical'); +$node_subscriber->append_conf( + 'postgresql.conf', qq[ +max_prepared_transactions = 10 +log_min_messages = DEBUG1 +]); +$node_subscriber->start; + +# Setup structures on the publisher and the subscriber +$node_publisher->safe_psql('postgres', "CREATE TABLE test_tab (a int)"); +$node_subscriber->safe_psql('postgres', + "CREATE TABLE test_tab (a int primary key);"); + +# Setup logical replication +my $publisher_connstr = $node_publisher->connstr . ' dbname=postgres'; +$node_publisher->safe_psql('postgres', + "CREATE PUBLICATION tap_pub FOR TABLE test_tab"); + +my $appname = 'tap_sub'; +my $subopts = 'streaming = on, two_phase = on, copy_data = false'; +$node_subscriber->safe_psql( + 'postgres', qq[ +CREATE SUBSCRIPTION tap_sub +CONNECTION '$publisher_connstr application_name=$appname' +PUBLICATION tap_pub WITH ($subopts); +]); + +$node_publisher->wait_for_catchup($appname); + +# There's no entry at the beginning +my $result = $node_subscriber->safe_psql('postgres', + "SELECT count(*) FROM pg_stat_subscription_workers;"); +is($result, q(0), 'no entry for transaction stats yet'); + +# COMMIT +$node_publisher->safe_psql('postgres', + "BEGIN; INSERT INTO test_tab VALUES (1); COMMIT;"); + +$node_subscriber->poll_query_until('postgres', + "SELECT count(1) = 1 FROM pg_stat_subscription_workers where commit_count = 1;" +) or die "didn't get updates of xact stats by commit"; + +# Now, stats collector make the bytes updated also. +$result = $node_subscriber->safe_psql('postgres', + "SELECT commit_bytes > 0 FROM pg_stat_subscription_workers;"); +is($result, q(t), 'got consumed bytes'); + +# STREAM COMMIT +$node_publisher->safe_psql('postgres', + "BEGIN; INSERT INTO test_tab VALUES(generate_series(1001, 2000)); COMMIT;" +); +$node_subscriber->poll_query_until('postgres', + "SELECT count(1) = 1 FROM pg_stat_subscription_workers where commit_count = 2;" +) or die "didn't get updates of xact stats by stream commit"; + +# STREAM PREPARE & COMMIT PREPARED +# This should restore the xact size before the shutdown. +$node_publisher->safe_psql( + 'postgres', q[ +BEGIN; +INSERT INTO test_tab VALUES(generate_series(2001, 3000)); +PREPARE TRANSACTION 'gid1'; +]); + +# This streamed prepare is not displayed until the commit prepared +# or rollback prepared. Hence, there's no way to confirm that +# stats collector has received the bytes of prepared transaction. +# So, instead of checking the view, issue one more committed transaction +# after the prepare and make sure that this commit's update is done, +# which should mean the previous streamed prepare is already processed +# by the stats collector as well. +$node_publisher->safe_psql('postgres', + "BEGIN; INSERT INTO test_tab VALUES (2); COMMIT;"); +$node_subscriber->poll_query_until('postgres', + "SELECT count(1) = 1 FROM pg_stat_subscription_workers where commit_count = 3;" +) or die "didn't process the updates of committed transaction"; + +$node_subscriber->restart; + +# Get commit_bytes before commit prepared. +my $tmp = $node_subscriber->safe_psql('postgres', + "SELECT commit_bytes FROM pg_stat_subscription_workers where commit_count = 3;" +); + +# Commit prepared increments the commit_count. +$node_publisher->safe_psql('postgres', "COMMIT PREPARED 'gid1'"); +$node_subscriber->poll_query_until('postgres', + "SELECT count(1) = 1 FROM pg_stat_subscription_workers where commit_count = 4;" + ) + or die + "didn't get updates of xact stats by stream prepare and commit prepared"; + +$node_subscriber->poll_query_until('postgres', + "SELECT commit_bytes > $tmp FROM pg_stat_subscription_workers where commit_count = 4;" +); + +# ROLLBACK PREPARED +$node_publisher->safe_psql( + 'postgres', q[ +BEGIN; +INSERT INTO test_tab VALUES (3); +PREPARE TRANSACTION 'gid2'; +]); +$node_publisher->safe_psql('postgres', "ROLLBACK PREPARED 'gid2'"); +$node_subscriber->poll_query_until('postgres', + "SELECT count(1) = 1 FROM pg_stat_subscription_workers where abort_count = 1;" +) or die "didn't get updates of xact stats by rollback prepared"; + +# error stats (by duplication error) +$node_publisher->safe_psql( + 'postgres', q[ +BEGIN; +INSERT INTO test_tab VALUES (1); +COMMIT; +]); + +$node_subscriber->poll_query_until('postgres', + "SELECT count(1) = 1 FROM pg_stat_subscription_workers where error_count > 0;" +) or die "didn't get updates of xact stats by error"; + +$node_subscriber->stop('fast'); +$node_publisher->stop('fast'); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f41ef0d..3403667 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -1945,6 +1945,8 @@ PgStat_MsgResetslrucounter PgStat_MsgSLRU PgStat_MsgSubscriptionPurge PgStat_MsgSubWorkerError +PgStat_MsgSubWorkerTwophaseXact +PgStat_MsgSubWorkerXactEnd PgStat_MsgTabpurge PgStat_MsgTabstat PgStat_MsgTempFile @@ -1958,6 +1960,8 @@ PgStat_StatFuncEntry PgStat_StatReplSlotEntry PgStat_StatSubWorkerEntry PgStat_StatSubWorkerKey +PgStat_SW_PreparedXactKey +PgStat_SW_PreparedXactEntry PgStat_StatTabEntry PgStat_SubXactStatus PgStat_TableCounts -- 2.2.0 ^ permalink raw reply [nested|flat] 113+ messages in thread
* RE: Failed transaction statistics to measure the logical replication progress @ 2021-11-16 12:34 [email protected] <[email protected]> parent: [email protected] <[email protected]> 0 siblings, 2 replies; 113+ messages in thread From: [email protected] @ 2021-11-16 12:34 UTC (permalink / raw) To: 'vignesh C' <[email protected]>; +Cc: Greg Nancarrow <[email protected]>; Masahiko Sawada <[email protected]>; Amit Kapila <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]> On Monday, November 15, 2021 9:14 PM I wrote: > I've conducted some update for this. > (The rebased part is only C code and checked by pgindent) I'll update my patches since a new skip xid patch has been shared in [1]. This version includes some minor renames of functions that are related to transaction sizes. [1] - https://www.postgresql.org/message-id/CAD21AoA5jupM6O%3DpYsyfaxQ1aMX-en8%3DQNgpW6KfXsg7_CS0CQ%40mail... Best Regards, Takamichi Osumi Attachments: [application/octet-stream] v12-0001-Rename-existing-columns-of-pg_stat_subscription_.patch (15.4K, ../../TYCPR01MB8373CFDCD60CCDB1C35CFA45ED999@TYCPR01MB8373.jpnprd01.prod.outlook.com/2-v12-0001-Rename-existing-columns-of-pg_stat_subscription_.patch) download | inline diff: From ceada4d0fe9ad536741a87c765171293c3fa8f36 Mon Sep 17 00:00:00 2001 From: Osumi Takamichi <[email protected]> Date: Mon, 15 Nov 2021 08:45:55 +0000 Subject: [PATCH v12 1/2] Rename existing columns of pg_stat_subscription_workers and export PartitionTupleRouting --- doc/src/sgml/monitoring.sgml | 24 +++++----- src/backend/catalog/system_views.sql | 10 ++--- src/backend/executor/execPartition.c | 70 ----------------------------- src/backend/utils/adt/pgstatfuncs.c | 20 ++++----- src/include/catalog/pg_proc.dat | 2 +- src/include/executor/execPartition.h | 69 ++++++++++++++++++++++++++++ src/test/regress/expected/rules.out | 12 ++--- src/test/subscription/t/026_error_report.pl | 8 ++-- 8 files changed, 107 insertions(+), 108 deletions(-) diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml index daf9fe8..5b10d18 100644 --- a/doc/src/sgml/monitoring.sgml +++ b/doc/src/sgml/monitoring.sgml @@ -3103,31 +3103,31 @@ SELECT pid, wait_event_type, wait_event FROM pg_stat_activity WHERE wait_event i <row> <entry role="catalog_table_entry"><para role="column_definition"> - <structfield>relid</structfield> <type>oid</type> + <structfield>last_error_relid</structfield> <type>oid</type> </para> <para> OID of the relation that the worker was processing when the - error occurred + last error occurred </para></entry> </row> <row> <entry role="catalog_table_entry"><para role="column_definition"> - <structfield>command</structfield> <type>text</type> + <structfield>last_error_command</structfield> <type>text</type> </para> <para> - Name of command being applied when the error occurred. This field - is always NULL if the error was reported during the initial data - copy. + Name of the last command being applied when the error occurred. + This field is always NULL if the error was reported during the + initial data copy. </para></entry> </row> <row> <entry role="catalog_table_entry"><para role="column_definition"> - <structfield>xid</structfield> <type>xid</type> + <structfield>last_error_xid</structfield> <type>xid</type> </para> <para> - Transaction ID of the publisher node being applied when the error + Transaction ID of the publisher node being applied when the last error occurred. This field is always NULL if the error was reported during the initial data copy. </para></entry> @@ -3135,19 +3135,19 @@ SELECT pid, wait_event_type, wait_event FROM pg_stat_activity WHERE wait_event i <row> <entry role="catalog_table_entry"><para role="column_definition"> - <structfield>error_count</structfield> <type>uint8</type> + <structfield>last_error_count</structfield> <type>uint8</type> </para> <para> - Number of consecutive times the error occurred + Number of consecutive times the last error occurred </para></entry> </row> <row> <entry role="catalog_table_entry"><para role="column_definition"> - <structfield>error_message</structfield> <type>text</type> + <structfield>last_error_message</structfield> <type>text</type> </para> <para> - The error message + The last error message </para></entry> </row> diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index cb2f77c..ecf1a0b 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1267,11 +1267,11 @@ CREATE VIEW pg_stat_subscription_workers AS w.subid, s.subname, w.subrelid, - w.relid, - w.command, - w.xid, - w.error_count, - w.error_message, + w.last_error_relid, + w.last_error_command, + w.last_error_xid, + w.last_error_count, + w.last_error_message, w.first_error_time, w.last_error_time FROM (SELECT diff --git a/src/backend/executor/execPartition.c b/src/backend/executor/execPartition.c index 5c723bc..2467f4c 100644 --- a/src/backend/executor/execPartition.c +++ b/src/backend/executor/execPartition.c @@ -34,76 +34,6 @@ #include "utils/rls.h" #include "utils/ruleutils.h" - -/*----------------------- - * PartitionTupleRouting - Encapsulates all information required to - * route a tuple inserted into a partitioned table to one of its leaf - * partitions. - * - * partition_root - * The partitioned table that's the target of the command. - * - * partition_dispatch_info - * Array of 'max_dispatch' elements containing a pointer to a - * PartitionDispatch object for every partitioned table touched by tuple - * routing. The entry for the target partitioned table is *always* - * present in the 0th element of this array. See comment for - * PartitionDispatchData->indexes for details on how this array is - * indexed. - * - * nonleaf_partitions - * Array of 'max_dispatch' elements containing pointers to fake - * ResultRelInfo objects for nonleaf partitions, useful for checking - * the partition constraint. - * - * num_dispatch - * The current number of items stored in the 'partition_dispatch_info' - * array. Also serves as the index of the next free array element for - * new PartitionDispatch objects that need to be stored. - * - * max_dispatch - * The current allocated size of the 'partition_dispatch_info' array. - * - * partitions - * Array of 'max_partitions' elements containing a pointer to a - * ResultRelInfo for every leaf partition touched by tuple routing. - * Some of these are pointers to ResultRelInfos which are borrowed out of - * the owning ModifyTableState node. The remainder have been built - * especially for tuple routing. See comment for - * PartitionDispatchData->indexes for details on how this array is - * indexed. - * - * is_borrowed_rel - * Array of 'max_partitions' booleans recording whether a given entry - * in 'partitions' is a ResultRelInfo pointer borrowed from the owning - * ModifyTableState node, rather than being built here. - * - * num_partitions - * The current number of items stored in the 'partitions' array. Also - * serves as the index of the next free array element for new - * ResultRelInfo objects that need to be stored. - * - * max_partitions - * The current allocated size of the 'partitions' array. - * - * memcxt - * Memory context used to allocate subsidiary structs. - *----------------------- - */ -struct PartitionTupleRouting -{ - Relation partition_root; - PartitionDispatch *partition_dispatch_info; - ResultRelInfo **nonleaf_partitions; - int num_dispatch; - int max_dispatch; - ResultRelInfo **partitions; - bool *is_borrowed_rel; - int num_partitions; - int max_partitions; - MemoryContext memcxt; -}; - /*----------------------- * PartitionDispatch - information about one partitioned table in a partition * hierarchy required to route a tuple to any of its partitions. A diff --git a/src/backend/utils/adt/pgstatfuncs.c b/src/backend/utils/adt/pgstatfuncs.c index b19729d..a34c0b6 100644 --- a/src/backend/utils/adt/pgstatfuncs.c +++ b/src/backend/utils/adt/pgstatfuncs.c @@ -2441,15 +2441,15 @@ pg_stat_get_subscription_worker(PG_FUNCTION_ARGS) OIDOID, -1, 0); TupleDescInitEntry(tupdesc, (AttrNumber) 2, "subrelid", OIDOID, -1, 0); - TupleDescInitEntry(tupdesc, (AttrNumber) 3, "relid", + TupleDescInitEntry(tupdesc, (AttrNumber) 3, "last_error_relid", OIDOID, -1, 0); - TupleDescInitEntry(tupdesc, (AttrNumber) 4, "command", + TupleDescInitEntry(tupdesc, (AttrNumber) 4, "last_error_command", TEXTOID, -1, 0); - TupleDescInitEntry(tupdesc, (AttrNumber) 5, "xid", + TupleDescInitEntry(tupdesc, (AttrNumber) 5, "last_error_xid", XIDOID, -1, 0); - TupleDescInitEntry(tupdesc, (AttrNumber) 6, "error_count", + TupleDescInitEntry(tupdesc, (AttrNumber) 6, "last_error_count", INT8OID, -1, 0); - TupleDescInitEntry(tupdesc, (AttrNumber) 7, "error_message", + TupleDescInitEntry(tupdesc, (AttrNumber) 7, "last_error_message", TEXTOID, -1, 0); TupleDescInitEntry(tupdesc, (AttrNumber) 8, "first_error_time", TIMESTAMPTZOID, -1, 0); @@ -2471,28 +2471,28 @@ pg_stat_get_subscription_worker(PG_FUNCTION_ARGS) else nulls[i++] = true; - /* relid */ + /* last_error_relid */ if (OidIsValid(wentry->relid)) values[i++] = ObjectIdGetDatum(wentry->relid); else nulls[i++] = true; - /* command */ + /* last_error_command */ if (wentry->command != 0) values[i++] = CStringGetTextDatum(logicalrep_message_type(wentry->command)); else nulls[i++] = true; - /* xid */ + /* last_error_xid */ if (TransactionIdIsValid(wentry->xid)) values[i++] = TransactionIdGetDatum(wentry->xid); else nulls[i++] = true; - /* error_count */ + /* last_error_count */ values[i++] = Int64GetDatum(wentry->error_count); - /* error_message */ + /* last_error_message */ values[i++] = CStringGetTextDatum(wentry->error_message); /* first_error_time */ diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index 50e1c7b..25f685f 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -5391,7 +5391,7 @@ prorettype => 'record', proargtypes => 'oid oid', proallargtypes => '{oid,oid,oid,oid,oid,text,xid,int8,text,timestamptz,timestamptz}', proargmodes => '{i,i,o,o,o,o,o,o,o,o,o}', - proargnames => '{subid,subrelid,subid,subrelid,relid,command,xid,error_count,error_message,first_error_time,last_error_time}', + proargnames => '{subid,subrelid,subid,subrelid,last_error_relid,last_error_command,last_error_xid,last_error_count,last_error_message,first_error_time,last_error_time}', prosrc => 'pg_stat_get_subscription_worker' }, { oid => '6118', descr => 'statistics: information about subscription', proname => 'pg_stat_get_subscription', prorows => '10', proisstrict => 'f', diff --git a/src/include/executor/execPartition.h b/src/include/executor/execPartition.h index 694e38b..4c796ab 100644 --- a/src/include/executor/execPartition.h +++ b/src/include/executor/execPartition.h @@ -110,6 +110,75 @@ typedef struct PartitionPruneState PartitionPruningData *partprunedata[FLEXIBLE_ARRAY_MEMBER]; } PartitionPruneState; +/*----------------------- + * PartitionTupleRouting - Encapsulates all information required to + * route a tuple inserted into a partitioned table to one of its leaf + * partitions. + * + * partition_root + * The partitioned table that's the target of the command. + * + * partition_dispatch_info + * Array of 'max_dispatch' elements containing a pointer to a + * PartitionDispatch object for every partitioned table touched by tuple + * routing. The entry for the target partitioned table is *always* + * present in the 0th element of this array. See comment for + * PartitionDispatchData->indexes for details on how this array is + * indexed. + * + * nonleaf_partitions + * Array of 'max_dispatch' elements containing pointers to fake + * ResultRelInfo objects for nonleaf partitions, useful for checking + * the partition constraint. + * + * num_dispatch + * The current number of items stored in the 'partition_dispatch_info' + * array. Also serves as the index of the next free array element for + * new PartitionDispatch objects that need to be stored. + * + * max_dispatch + * The current allocated size of the 'partition_dispatch_info' array. + * + * partitions + * Array of 'max_partitions' elements containing a pointer to a + * ResultRelInfo for every leaf partition touched by tuple routing. + * Some of these are pointers to ResultRelInfos which are borrowed out of + * the owning ModifyTableState node. The remainder have been built + * especially for tuple routing. See comment for + * PartitionDispatchData->indexes for details on how this array is + * indexed. + * + * is_borrowed_rel + * Array of 'max_partitions' booleans recording whether a given entry + * in 'partitions' is a ResultRelInfo pointer borrowed from the owning + * ModifyTableState node, rather than being built here. + * + * num_partitions + * The current number of items stored in the 'partitions' array. Also + * serves as the index of the next free array element for new + * ResultRelInfo objects that need to be stored. + * + * max_partitions + * The current allocated size of the 'partitions' array. + * + * memcxt + * Memory context used to allocate subsidiary structs. + *----------------------- + */ +struct PartitionTupleRouting +{ + Relation partition_root; + PartitionDispatch *partition_dispatch_info; + ResultRelInfo **nonleaf_partitions; + int num_dispatch; + int max_dispatch; + ResultRelInfo **partitions; + bool *is_borrowed_rel; + int num_partitions; + int max_partitions; + MemoryContext memcxt; +}; + extern PartitionTupleRouting *ExecSetupPartitionTupleRouting(EState *estate, Relation rel); extern ResultRelInfo *ExecFindPartition(ModifyTableState *mtstate, diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out index cb6da2c..d60c5a5 100644 --- a/src/test/regress/expected/rules.out +++ b/src/test/regress/expected/rules.out @@ -2097,11 +2097,11 @@ pg_stat_subscription| SELECT su.oid AS subid, pg_stat_subscription_workers| SELECT w.subid, s.subname, w.subrelid, - w.relid, - w.command, - w.xid, - w.error_count, - w.error_message, + w.last_error_relid, + w.last_error_command, + w.last_error_xid, + w.last_error_count, + w.last_error_message, w.first_error_time, w.last_error_time FROM ( SELECT pg_subscription.oid AS subid, @@ -2112,7 +2112,7 @@ pg_stat_subscription_workers| SELECT w.subid, pg_subscription_rel.srrelid AS relid FROM pg_subscription_rel WHERE (pg_subscription_rel.srsubstate <> 'r'::"char")) sr, - (LATERAL pg_stat_get_subscription_worker(sr.subid, sr.relid) w(subid, subrelid, relid, command, xid, error_count, error_message, first_error_time, last_error_time) + (LATERAL pg_stat_get_subscription_worker(sr.subid, sr.relid) w(subid, subrelid, last_error_relid, last_error_command, last_error_xid, last_error_count, last_error_message, first_error_time, last_error_time) JOIN pg_subscription s ON ((w.subid = s.oid))); pg_stat_sys_indexes| SELECT pg_stat_all_indexes.relid, pg_stat_all_indexes.indexrelid, diff --git a/src/test/subscription/t/026_error_report.pl b/src/test/subscription/t/026_error_report.pl index 1227654..ca30ab2 100644 --- a/src/test/subscription/t/026_error_report.pl +++ b/src/test/subscription/t/026_error_report.pl @@ -15,8 +15,8 @@ sub test_subscription_error my $check_sql = qq[ SELECT count(1) > 0 FROM pg_stat_subscription_workers -WHERE relid = '$relname'::regclass]; - $check_sql .= " AND xid = '$xid'::xid;" if $xid ne ''; +WHERE last_error_relid = '$relname'::regclass]; + $check_sql .= " AND last_error_xid = '$xid'::xid;" if $xid ne ''; # Wait for the error statistics to be updated. $node->poll_query_until( @@ -26,9 +26,9 @@ WHERE relid = '$relname'::regclass]; my $result = $node->safe_psql( 'postgres', qq[ -SELECT subname, command, relid::regclass, error_count > 0 +SELECT subname, last_error_command, last_error_relid::regclass, last_error_count > 0 FROM pg_stat_subscription_workers -WHERE relid = '$relname'::regclass; +WHERE last_error_relid = '$relname'::regclass; ]); is($result, $expected_error, $msg); } -- 2.2.0 [application/octet-stream] v12-0002-Extend-pg_stat_subscription_workers-to-include-g.patch (53.0K, ../../TYCPR01MB8373CFDCD60CCDB1C35CFA45ED999@TYCPR01MB8373.jpnprd01.prod.outlook.com/3-v12-0002-Extend-pg_stat_subscription_workers-to-include-g.patch) download | inline diff: From 9e5a4eda23ca23dcc33f90139e11dd27761763c1 Mon Sep 17 00:00:00 2001 From: Osumi Takamichi <[email protected]> Date: Tue, 16 Nov 2021 12:01:54 +0000 Subject: [PATCH v12 2/2] Extend pg_stat_subscription_workers to include general transaction statistics Categorize transactions of logical replication subscriber into three types (commit, error, abort) and introduce cumulative columns of those numbers and amounts of consumed data during message apply respectively. The calculation of consumed resources by subscriber's apply is computed based on the data structure for message apply and extra data, which is different from that of publisher's decoding processing. At present, there's no special consideration to spool file statistics such as amount of data spooled to disk or its count. But, this can be added later. The amount of data used by apply for STREAM COMMIT and STREAM PREPARE is transaction data consumption and can be regarded as data that should be added to column of this commit. This size of prepared transaction becomes persistent to conclude the appropriate category for the transaction at either commit prepared or rollback prepared time. Author: Takamichi Osumi Discussed & Reviewed-by: Amit Kapila, Masahiko Sawada, Hou Zhijie, Greg Nancarrow, Vignesh C, Ajin Cherian Discussion: https://www.postgresql.org/message-id/OSBPR01MB48887CA8F40C8D984A6DC00CED199%40OSBPR01MB4888.jpnprd01.prod.outlook.com --- doc/src/sgml/monitoring.sgml | 71 ++++- src/backend/catalog/system_views.sql | 6 + src/backend/postmaster/pgstat.c | 324 ++++++++++++++++++++++- src/backend/replication/logical/proto.c | 12 + src/backend/replication/logical/tablesync.c | 6 + src/backend/replication/logical/worker.c | 174 +++++++++++- src/backend/utils/adt/pgstatfuncs.c | 36 ++- src/include/catalog/pg_proc.dat | 6 +- src/include/pgstat.h | 101 ++++++- src/include/replication/logicalworker.h | 5 + src/test/regress/expected/rules.out | 8 +- src/test/subscription/t/027_worker_xact_stats.pl | 146 ++++++++++ src/tools/pgindent/typedefs.list | 4 + 13 files changed, 878 insertions(+), 21 deletions(-) create mode 100644 src/test/subscription/t/027_worker_xact_stats.pl diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml index 5b10d18..22caeef 100644 --- a/doc/src/sgml/monitoring.sgml +++ b/doc/src/sgml/monitoring.sgml @@ -629,8 +629,8 @@ postgres 27093 0.0 0.0 30096 2752 ? Ss 11:34 0:00 postgres: ser <row> <entry><structname>pg_stat_subscription_workers</structname><indexterm><primary>pg_stat_subscription_workers</primary></indexterm></entry> - <entry>One row per subscription worker, showing statistics about errors - that occurred on that subscription worker. + <entry>One row per subscription worker, showing transaction statistics + and information about errors that occurred on that subscription worker. See <link linkend="monitoring-pg-stat-subscription-workers"> <structname>pg_stat_subscription_workers</structname></link> for details. </entry> @@ -3052,10 +3052,12 @@ SELECT pid, wait_event_type, wait_event FROM pg_stat_activity WHERE wait_event i <para> The <structname>pg_stat_subscription_workers</structname> view will contain - one row per subscription error reported by workers applying logical + one row per subscription, showing corresponding transaction statistics and + information about the error reported by workers applying logical replication changes and workers handling the initial data copy of the subscribed tables. The statistics entry is removed when the subscription - the worker is running on is removed. + the worker is running on is removed. The statistics of transaction size is + utilized only by the apply worker. </para> <table id="pg-stat-subscription-workers" xreflabel="pg_stat_subscription_workers"> @@ -3103,6 +3105,67 @@ SELECT pid, wait_event_type, wait_event FROM pg_stat_activity WHERE wait_event i <row> <entry role="catalog_table_entry"><para role="column_definition"> + <structfield>commit_count</structfield> <type>bigint</type> + </para> + <para> + Number of transactions successfully applied in this subscription. + COMMIT, COMMIT of streaming transaction and COMMIT PREPARED increments + this counter. + </para></entry> + </row> + + <row> + <entry role="catalog_table_entry"><para role="column_definition"> + <structfield>commit_bytes</structfield> <type>bigint</type> + </para> + <para> + Amount of data (in bytes) successfully applied in this subscription, + across <literal>commit_count</literal> transactions. + </para></entry> + </row> + + <row> + <entry role="catalog_table_entry"><para role="column_definition"> + <structfield>error_count</structfield> <type>bigint</type> + </para> + <para> + Number of transactions that failed to be applied by the table + sync worker or main apply worker in this subscription. + </para></entry> + </row> + + <row> + <entry role="catalog_table_entry"><para role="column_definition"> + <structfield>error_bytes</structfield> <type>bigint</type> + </para> + <para> + Amount of data (in bytes) unsuccessfully applied in this subscription, + across <literal>error_count</literal> transactions. + </para></entry> + </row> + + <row> + <entry role="catalog_table_entry"><para role="column_definition"> + <structfield>abort_count</structfield> <type>bigint</type> + </para> + <para> + Number of transactions aborted in this subscription. + ROLLBACK PREPARED increments this counter. + </para></entry> + </row> + + <row> + <entry role="catalog_table_entry"><para role="column_definition"> + <structfield>abort_bytes</structfield> <type>bigint</type> + </para> + <para> + Amount of data (in bytes) aborted in this subscription, + across <literal>abort_count</literal> transactions. + </para></entry> + </row> + + <row> + <entry role="catalog_table_entry"><para role="column_definition"> <structfield>last_error_relid</structfield> <type>oid</type> </para> <para> diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index ecf1a0b..a8880e8 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1267,6 +1267,12 @@ CREATE VIEW pg_stat_subscription_workers AS w.subid, s.subname, w.subrelid, + w.commit_count, + w.commit_bytes, + w.error_count, + w.error_bytes, + w.abort_count, + w.abort_bytes, w.last_error_relid, w.last_error_command, w.last_error_xid, diff --git a/src/backend/postmaster/pgstat.c b/src/backend/postmaster/pgstat.c index ee3b39a..d0e4ee0 100644 --- a/src/backend/postmaster/pgstat.c +++ b/src/backend/postmaster/pgstat.c @@ -53,6 +53,7 @@ #include "postmaster/fork_process.h" #include "postmaster/interrupt.h" #include "postmaster/postmaster.h" +#include "replication/logicalworker.h" #include "replication/slot.h" #include "replication/walsender.h" #include "storage/backendid.h" @@ -325,11 +326,14 @@ static PgStat_StatTabEntry *pgstat_get_tab_entry(PgStat_StatDBEntry *dbentry, static PgStat_StatSubWorkerEntry *pgstat_get_subworker_entry(PgStat_StatDBEntry *dbentry, Oid subid, Oid subrelid, bool create); +static PgStat_SW_PreparedXactEntry *pgstat_get_subworker_prepared_txn(Oid databaseid, Oid subid, + char *gid, bool create); + static void pgstat_write_statsfiles(bool permanent, bool allDbs); static void pgstat_write_db_statsfile(PgStat_StatDBEntry *dbentry, bool permanent); static HTAB *pgstat_read_statsfiles(Oid onlydb, bool permanent, bool deep); static void pgstat_read_db_statsfile(Oid databaseid, HTAB *tabhash, HTAB *funchash, - HTAB *subworkerhash, bool permanent); + HTAB *subworkerhash, HTAB *preparedtxnhash, bool permanent); static void backend_read_statsfile(void); static bool pgstat_write_statsfile_needed(void); @@ -382,6 +386,9 @@ static void pgstat_recv_replslot(PgStat_MsgReplSlot *msg, int len); static void pgstat_recv_tempfile(PgStat_MsgTempFile *msg, int len); static void pgstat_recv_subscription_purge(PgStat_MsgSubscriptionPurge *msg, int len); static void pgstat_recv_subworker_error(PgStat_MsgSubWorkerError *msg, int len); +static void pgstat_recv_subworker_xact_end(PgStat_MsgSubWorkerXactEnd *msg, int len); +static void pgstat_recv_subworker_twophase_xact(PgStat_MsgSubWorkerTwophaseXact *msg, int len); + /* ------------------------------------------------------------ * Public functions called from postmaster follow @@ -1930,6 +1937,61 @@ pgstat_report_replslot_drop(const char *slotname) } /* ---------- + * pgstat_report_subworker_xact_end() - + * + * Tell the collector that worker transaction has successfully completed. + * ---------- + */ +void +pgstat_report_subworker_xact_end(Oid subid, Oid subrel, + LogicalRepMsgType command, PgStat_Counter xact_size) +{ + PgStat_MsgSubWorkerXactEnd msg; + + Assert(command == 0 /* table sync worker */ || + command == LOGICAL_REP_MSG_COMMIT || + command == LOGICAL_REP_MSG_STREAM_COMMIT); + + pgstat_setheader(&msg.m_hdr, PGSTAT_MTYPE_SUBWORKERXACTEND); + msg.m_databaseid = MyDatabaseId; + msg.m_subid = subid; + msg.m_subrelid = subrel; + msg.m_command = command; + msg.m_xact_bytes = xact_size; + pgstat_send(&msg, sizeof(PgStat_MsgSubWorkerXactEnd)); + + reset_apply_xact_size(); +} + +/* ---------- + * pgstat_report_subworker_twophase_xact() - + * + * Tell the collector that worker transaction has done 2PC related operation. + * ---------- + */ +void +pgstat_report_subworker_twophase_xact(Oid subid, LogicalRepMsgType command, + PgStat_Counter xact_size, char *gid) +{ + PgStat_MsgSubWorkerTwophaseXact msg; + + Assert(command == LOGICAL_REP_MSG_PREPARE || + command == LOGICAL_REP_MSG_STREAM_PREPARE || + command == LOGICAL_REP_MSG_COMMIT_PREPARED || + command == LOGICAL_REP_MSG_ROLLBACK_PREPARED); + + /* setup the message */ + pgstat_setheader(&msg.m_hdr, PGSTAT_MTYPE_SUBWORKERTWOPHASEXACT); + msg.m_databaseid = MyDatabaseId; + msg.m_subid = subid; + msg.m_command = command; + strlcpy(msg.m_gid, gid, sizeof(msg.m_gid)); + msg.m_xact_bytes = xact_size; + pgstat_send(&msg, sizeof(PgStat_MsgSubWorkerTwophaseXact)); + reset_apply_xact_size(); +} + +/* ---------- * pgstat_report_subworker_error() - * * Tell the collector about the subscription worker error. @@ -1947,6 +2009,7 @@ pgstat_report_subworker_error(Oid subid, Oid subrelid, Oid relid, msg.m_databaseid = MyDatabaseId; msg.m_subid = subid; msg.m_subrelid = subrelid; + msg.m_xact_error_bytes = get_apply_xact_size(); msg.m_relid = relid; msg.m_command = command; msg.m_xid = xid; @@ -1955,6 +2018,8 @@ pgstat_report_subworker_error(Oid subid, Oid subrelid, Oid relid, len = offsetof(PgStat_MsgSubWorkerError, m_message) + strlen(msg.m_message) + 1; pgstat_send(&msg, len); + + reset_apply_xact_size(); } /* ---------- @@ -3725,6 +3790,14 @@ PgstatCollectorMain(int argc, char *argv[]) pgstat_recv_subworker_error(&msg.msg_subworkererror, len); break; + case PGSTAT_MTYPE_SUBWORKERXACTEND: + pgstat_recv_subworker_xact_end(&msg.msg_subworkerxactend, len); + break; + + case PGSTAT_MTYPE_SUBWORKERTWOPHASEXACT: + pgstat_recv_subworker_twophase_xact(&msg.msg_subworkertwophasexact, len); + break; + default: break; } @@ -3831,6 +3904,14 @@ reset_dbentry_counters(PgStat_StatDBEntry *dbentry) PGSTAT_SUBWORKER_HASH_SIZE, &hash_ctl, HASH_ELEM | HASH_BLOBS); + + hash_ctl.keysize = sizeof(PgStat_SW_PreparedXactKey); + hash_ctl.entrysize = sizeof(PgStat_SW_PreparedXactEntry); + hash_ctl.hcxt = pgStatLocalContext; + dbentry->subworkers_preparedsizes = hash_create("Subscription worker stats of prepared txn", + PGSTAT_SUBWORKER_HASH_SIZE, + &hash_ctl, + HASH_ELEM | HASH_STRINGS | HASH_CONTEXT); } /* @@ -3941,6 +4022,12 @@ pgstat_get_subworker_entry(PgStat_StatDBEntry *dbentry, Oid subid, Oid subrelid, /* If not found, initialize the new one */ if (create && !found) { + subwentry->xact_commit_count = 0; + subwentry->xact_commit_bytes = 0; + subwentry->xact_error_count = 0; + subwentry->xact_error_bytes = 0; + subwentry->xact_abort_count = 0; + subwentry->xact_abort_bytes = 0; subwentry->relid = InvalidOid; subwentry->command = 0; subwentry->xid = InvalidTransactionId; @@ -4152,9 +4239,11 @@ pgstat_write_db_statsfile(PgStat_StatDBEntry *dbentry, bool permanent) HASH_SEQ_STATUS tstat; HASH_SEQ_STATUS fstat; HASH_SEQ_STATUS sstat; + HASH_SEQ_STATUS pstat; PgStat_StatTabEntry *tabentry; PgStat_StatFuncEntry *funcentry; PgStat_StatSubWorkerEntry *subwentry; + PgStat_SW_PreparedXactEntry *pentry; FILE *fpout; int32 format_id; Oid dbid = dbentry->databaseid; @@ -4221,6 +4310,17 @@ pgstat_write_db_statsfile(PgStat_StatDBEntry *dbentry, bool permanent) } /* + * Write subscription worker's prepared transaction struct + */ + hash_seq_init(&pstat, dbentry->subworkers_preparedsizes); + while ((pentry = (PgStat_SW_PreparedXactEntry *) hash_seq_search(&pstat)) != NULL) + { + fputc('P', fpout); + rc = fwrite(pentry, sizeof(PgStat_SW_PreparedXactEntry), 1, fpout); + (void) rc; /* we'll check for error with ferror */ + } + + /* * No more output to be done. Close the temp file and replace the old * pgstat.stat with it. The ferror() check replaces testing for error * after each individual fputc or fwrite above. @@ -4459,6 +4559,7 @@ pgstat_read_statsfiles(Oid onlydb, bool permanent, bool deep) dbentry->tables = NULL; dbentry->functions = NULL; dbentry->subworkers = NULL; + dbentry->subworkers_preparedsizes = NULL; /* * In the collector, disregard the timestamp we read from the @@ -4504,6 +4605,14 @@ pgstat_read_statsfiles(Oid onlydb, bool permanent, bool deep) &hash_ctl, HASH_ELEM | HASH_BLOBS | HASH_CONTEXT); + hash_ctl.keysize = sizeof(PgStat_SW_PreparedXactKey); + hash_ctl.entrysize = sizeof(PgStat_SW_PreparedXactEntry); + hash_ctl.hcxt = pgStatLocalContext; + dbentry->subworkers_preparedsizes = hash_create("Subscription worker stats of prepared txn", + PGSTAT_SUBWORKER_HASH_SIZE, + &hash_ctl, + HASH_ELEM | HASH_STRINGS | HASH_CONTEXT); + /* * If requested, read the data from the database-specific * file. Otherwise we just leave the hashtables empty. @@ -4513,6 +4622,7 @@ pgstat_read_statsfiles(Oid onlydb, bool permanent, bool deep) dbentry->tables, dbentry->functions, dbentry->subworkers, + dbentry->subworkers_preparedsizes, permanent); break; @@ -4597,7 +4707,7 @@ done: */ static void pgstat_read_db_statsfile(Oid databaseid, HTAB *tabhash, HTAB *funchash, - HTAB *subworkerhash, bool permanent) + HTAB *subworkerhash, HTAB *preparedtxnhash, bool permanent) { PgStat_StatTabEntry *tabentry; PgStat_StatTabEntry tabbuf; @@ -4753,6 +4863,32 @@ pgstat_read_db_statsfile(Oid databaseid, HTAB *tabhash, HTAB *funchash, memcpy(subwentry, &subwbuf, sizeof(subwbuf)); break; + case 'P': + { + PgStat_SW_PreparedXactEntry buff; + PgStat_SW_PreparedXactEntry *prepared_xact_entry; + + if (fread(&buff, 1, sizeof(PgStat_SW_PreparedXactEntry), + fpin) != sizeof(PgStat_SW_PreparedXactEntry)) + { + ereport(pgStatRunningInCollector ? LOG : WARNING, + (errmsg("corrupted statistics file \"%s\"", + statfile))); + goto done; + } + + if (preparedtxnhash == NULL) + break; + + prepared_xact_entry = + (PgStat_SW_PreparedXactEntry *) hash_search(preparedtxnhash, + (void *) &buff.key, + HASH_ENTER, NULL); + + memcpy(prepared_xact_entry, &buff, sizeof(PgStat_SW_PreparedXactEntry)); + break; + } + /* * 'E' The EOF marker of a complete stats file. */ @@ -5428,6 +5564,8 @@ pgstat_recv_dropdb(PgStat_MsgDropdb *msg, int len) hash_destroy(dbentry->functions); if (dbentry->subworkers != NULL) hash_destroy(dbentry->subworkers); + if (dbentry->subworkers_preparedsizes != NULL) + hash_destroy(dbentry->subworkers_preparedsizes); if (hash_search(pgStatDBHash, (void *) &dbid, @@ -5467,10 +5605,13 @@ pgstat_recv_resetcounter(PgStat_MsgResetcounter *msg, int len) hash_destroy(dbentry->functions); if (dbentry->subworkers != NULL) hash_destroy(dbentry->subworkers); + if (dbentry->subworkers_preparedsizes != NULL) + hash_destroy(dbentry->subworkers_preparedsizes); dbentry->tables = NULL; dbentry->functions = NULL; dbentry->subworkers = NULL; + dbentry->subworkers_preparedsizes = NULL; /* * Reset database-level stats, too. This creates empty hash tables for @@ -5546,10 +5687,28 @@ pgstat_recv_resetsinglecounter(PgStat_MsgResetsinglecounter *msg, int len) else if (msg->m_resettype == RESET_SUBWORKER) { PgStat_StatSubWorkerKey key; + PgStat_SW_PreparedXactKey pkey; key.subid = msg->m_objectid; key.subrelid = msg->m_subobjectid; (void) hash_search(dbentry->subworkers, (void *) &key, HASH_REMOVE, NULL); + + /* + * Clean up entry of prepared size hash as well. + * + * There's no gid input here as a part of key for + * PgStat_SW_PreparedXactEntry. Also any valid 'relid' indicates the + * entry is created by table sync but it has nothing to do with the + * two phase operation of apply. Proceed with this removal only when + * the specified subrelid isn't invalid. + */ + if (msg->m_subobjectid == InvalidOid) + { + pkey.subid = msg->m_objectid; + (void) hash_search(dbentry->subworkers_preparedsizes, + (void *) &pkey, HASH_REMOVE, NULL); + } + } } @@ -6105,6 +6264,7 @@ pgstat_recv_subscription_purge(PgStat_MsgSubscriptionPurge *msg, int len) HASH_SEQ_STATUS hstat; PgStat_StatDBEntry *dbentry; PgStat_StatSubWorkerEntry *subwentry; + PgStat_SW_PreparedXactEntry *prepared_txn_entry; dbentry = pgstat_get_db_entry(msg->m_databaseid, false); @@ -6126,6 +6286,127 @@ pgstat_recv_subscription_purge(PgStat_MsgSubscriptionPurge *msg, int len) } } } + + /* Remove associated prepared transaction stats */ + if (!dbentry->subworkers_preparedsizes) + { + hash_seq_init(&hstat, dbentry->subworkers_preparedsizes); + while ((prepared_txn_entry = (PgStat_SW_PreparedXactEntry *) hash_seq_search(&hstat)) != NULL) + { + for (int i = 0; i < msg->m_nentries; i++) + { + if (prepared_txn_entry->key.subid == msg->m_subids[i]) + (void) hash_search(dbentry->subworkers_preparedsizes, + (void *) &(prepared_txn_entry->key), + HASH_REMOVE, NULL); + break; + } + } + } +} + +/* ---------- + * pgstat_recv_subworker_xact_end() - + * + * Process a SUBWORKERXACTEND message. + * ---------- + */ +static void +pgstat_recv_subworker_xact_end(PgStat_MsgSubWorkerXactEnd *msg, int len) +{ + PgStat_StatDBEntry *dbentry; + PgStat_StatSubWorkerEntry *wentry; + + dbentry = pgstat_get_db_entry(msg->m_databaseid, true); + wentry = pgstat_get_subworker_entry(dbentry, msg->m_subid, + msg->m_subrelid, true); + Assert(wentry); + + /* table sync worker */ + if (msg->m_command == 0) + wentry->xact_commit_count++; + else + { + /* apply worker */ + switch (msg->m_command) + { + case LOGICAL_REP_MSG_COMMIT: + case LOGICAL_REP_MSG_STREAM_COMMIT: + wentry->xact_commit_count++; + wentry->xact_commit_bytes += msg->m_xact_bytes; + break; + default: + elog(ERROR, "unexpected logical message type as normal apply end"); + break; + } + } +} + +/* ---------- + * pgstat_recv_subworker_twophase_xact() - + * + * Process a SUBWORKERTWOPHASEXACT message. + * ---------- + */ +static void +pgstat_recv_subworker_twophase_xact(PgStat_MsgSubWorkerTwophaseXact *msg, int len) +{ + PgStat_StatDBEntry *dbentry; + PgStat_SW_PreparedXactEntry *pentry; + PgStat_StatSubWorkerEntry *wentry; + PgStat_SW_PreparedXactKey key; + + pentry = pgstat_get_subworker_prepared_txn(msg->m_databaseid, + msg->m_subid, + msg->m_gid, true); + Assert(pentry); + switch (msg->m_command) + { + case LOGICAL_REP_MSG_PREPARE: + case LOGICAL_REP_MSG_STREAM_PREPARE: + + /* + * Make each size of prepared transaction persistent so that we + * can update stats over the server restart and make prepared + * stats updated when commit prepared or rollback prepared + * arrives. + */ + pentry->subid = msg->m_subid; + strlcpy(pentry->gid, msg->m_gid, sizeof(pentry->gid)); + pentry->xact_size = msg->m_xact_bytes; + break; + + case LOGICAL_REP_MSG_COMMIT_PREPARED: + case LOGICAL_REP_MSG_ROLLBACK_PREPARED: + /* Update exported xact stats now */ + dbentry = pgstat_get_db_entry(msg->m_databaseid, true); + wentry = pgstat_get_subworker_entry(dbentry, + msg->m_subid, + InvalidOid /* apply worker */ , + true); + Assert(wentry); + if (msg->m_command == LOGICAL_REP_MSG_COMMIT_PREPARED) + { + wentry->xact_commit_count++; + wentry->xact_commit_bytes += pentry->xact_size; + } + else + { + wentry->xact_abort_count++; + wentry->xact_abort_bytes += pentry->xact_size; + } + + /* Clean up this gid from transaction size hash */ + key.subid = pentry->subid; + strlcpy(key.gid, msg->m_gid, sizeof(key.gid)); + (void) hash_search(dbentry->subworkers_preparedsizes, + (void *) &key, HASH_REMOVE, NULL); + break; + + default: + elog(ERROR, "unexpected logical message type as prepare transaction"); + break; + } } /* ---------- @@ -6147,6 +6428,10 @@ pgstat_recv_subworker_error(PgStat_MsgSubWorkerError *msg, int len) msg->m_subrelid, true); Assert(subwentry); + /* general transaction stats for error */ + subwentry->xact_error_count++; + subwentry->xact_error_bytes += msg->m_xact_error_bytes; + /* * Update only the counter and last error timestamp if we received * the same error again @@ -6394,3 +6679,38 @@ pgstat_count_slru_truncate(int slru_idx) { slru_entry(slru_idx)->m_truncate += 1; } + + /* ---------- + * pgstat_get_subworker_prepared_txn + * + * Return subscription worker entry with the given subscription OID and + * gid. + * ---------- + */ +static PgStat_SW_PreparedXactEntry * +pgstat_get_subworker_prepared_txn(Oid databaseid, Oid subid, + char *gid, bool create) +{ + PgStat_StatDBEntry *dbentry; + PgStat_SW_PreparedXactKey key; + PgStat_SW_PreparedXactEntry *pentry; + HASHACTION action; + bool found; + + dbentry = pgstat_get_db_entry(databaseid, true); + key.subid = subid; + strlcpy(key.gid, gid, sizeof(key.gid)); + action = (create ? HASH_ENTER : HASH_FIND); + pentry = (PgStat_SW_PreparedXactEntry *) hash_search(dbentry->subworkers_preparedsizes, + (void *) &key, + action, &found); + + if (create && !found) + { + pentry->subid = 0; + pentry->gid[0] = '\0'; + pentry->xact_size = 0; + } + + return pentry; +} diff --git a/src/backend/replication/logical/proto.c b/src/backend/replication/logical/proto.c index 9f5bf4b..8306876 100644 --- a/src/backend/replication/logical/proto.c +++ b/src/backend/replication/logical/proto.c @@ -17,6 +17,7 @@ #include "catalog/pg_type.h" #include "libpq/pqformat.h" #include "replication/logicalproto.h" +#include "replication/logicalworker.h" #include "utils/lsyscache.h" #include "utils/syscache.h" @@ -842,6 +843,7 @@ logicalrep_read_tuple(StringInfo in, LogicalRepTupleData *tuple) { int i; int natts; + Size total_size; /* Get number of attributes */ natts = pq_getmsgint(in, 2); @@ -850,6 +852,7 @@ logicalrep_read_tuple(StringInfo in, LogicalRepTupleData *tuple) tuple->colvalues = (StringInfoData *) palloc0(natts * sizeof(StringInfoData)); tuple->colstatus = (char *) palloc(natts * sizeof(char)); tuple->ncols = natts; + total_size = natts * (sizeof(char) + sizeof(StringInfoData)); /* Read the data */ for (i = 0; i < natts; i++) @@ -880,6 +883,9 @@ logicalrep_read_tuple(StringInfo in, LogicalRepTupleData *tuple) value->len = len; value->cursor = 0; value->maxlen = len; + + /* memory for tuple */ + total_size += len + 1; break; case LOGICALREP_COLUMN_BINARY: len = pq_getmsgint(in, 4); /* read length */ @@ -893,11 +899,17 @@ logicalrep_read_tuple(StringInfo in, LogicalRepTupleData *tuple) value->len = len; value->cursor = 0; value->maxlen = len; + + /* memory for tuple */ + total_size += len + 1; break; default: elog(ERROR, "unrecognized data representation type '%c'", kind); } } + + /* Record memory consumption for tuple */ + add_apply_xact_size(total_size); } /* diff --git a/src/backend/replication/logical/tablesync.c b/src/backend/replication/logical/tablesync.c index f07983a..5bb1d63 100644 --- a/src/backend/replication/logical/tablesync.c +++ b/src/backend/replication/logical/tablesync.c @@ -1149,6 +1149,12 @@ copy_table_done: MyLogicalRepWorker->relstate_lsn = *origin_startpos; SpinLockRelease(&MyLogicalRepWorker->relmutex); + /* Report the success of table sync. */ + pgstat_report_subworker_xact_end(MyLogicalRepWorker->subid, + MyLogicalRepWorker->relid, + 0 /* no logical message type */ , + 0 /* xact size */ ); + /* * Finally, wait until the main apply worker tells us to catch up and then * return to let LogicalRepApplyLoop do it. diff --git a/src/backend/replication/logical/worker.c b/src/backend/replication/logical/worker.c index 2e79302..ec9c270 100644 --- a/src/backend/replication/logical/worker.c +++ b/src/backend/replication/logical/worker.c @@ -221,6 +221,18 @@ typedef struct ApplyErrorCallbackArg LogicalRepMsgType command; /* 0 if invalid */ LogicalRepRelMapEntry *rel; + /* + * Store data size of this transaction. + * + * The byte size of transaction on the publisher is calculated by + * ReorderBufferChangeSize() based on the ReorderBufferChange structure. + * But on the subscriber, consumed resources are not same as the + * publisher's decoding processsing and required to be computed in + * different way. Therefore, the exact same byte size is not restored on + * the subscriber usually. + */ + PgStat_Counter bytes; + /* Remote node information */ int remote_attnum; /* -1 if invalid */ TransactionId remote_xid; @@ -231,6 +243,7 @@ static ApplyErrorCallbackArg apply_error_callback_arg = { .command = 0, .rel = NULL, + .bytes = 0, .remote_attnum = -1, .remote_xid = InvalidTransactionId, .ts = 0, @@ -293,6 +306,7 @@ static inline void cleanup_subxact_info(void); static void stream_cleanup_files(Oid subid, TransactionId xid); static void stream_open_file(Oid subid, TransactionId xid, bool first); static void stream_write_change(char action, StringInfo s); + static void stream_close_file(void); static void send_feedback(XLogRecPtr recvpos, bool force, bool requestReply); @@ -304,6 +318,9 @@ static void maybe_reread_subscription(void); /* prototype needed because of stream_commit */ static void apply_dispatch(StringInfo s); +static void update_apply_change_size(LogicalRepMsgType action, + bool is_partitioned); + static void apply_handle_commit_internal(LogicalRepCommitData *commit_data); static void apply_handle_insert_internal(ApplyExecutionData *edata, ResultRelInfo *relinfo, @@ -818,6 +835,11 @@ apply_handle_commit(StringInfo s) /* Process any tables that are being synchronized in parallel. */ process_syncing_tables(commit_data.end_lsn); + pgstat_report_subworker_xact_end(MyLogicalRepWorker->subid, + InvalidOid, + LOGICAL_REP_MSG_COMMIT, + get_apply_xact_size()); + pgstat_report_activity(STATE_IDLE, NULL); reset_apply_error_context_info(); } @@ -922,6 +944,13 @@ apply_handle_prepare(StringInfo s) /* Process any tables that are being synchronized in parallel. */ process_syncing_tables(prepare_data.end_lsn); + /* Update the memory consumption of prepare and report */ + update_apply_change_size(LOGICAL_REP_MSG_PREPARE, false); + pgstat_report_subworker_twophase_xact(MyLogicalRepWorker->subid, + LOGICAL_REP_MSG_PREPARE, + get_apply_xact_size(), + prepare_data.gid); + pgstat_report_activity(STATE_IDLE, NULL); reset_apply_error_context_info(); } @@ -963,6 +992,13 @@ apply_handle_commit_prepared(StringInfo s) /* Process any tables that are being synchronized in parallel. */ process_syncing_tables(prepare_data.end_lsn); + /* Update the transaction size and report */ + update_apply_change_size(LOGICAL_REP_MSG_COMMIT_PREPARED, false); + pgstat_report_subworker_twophase_xact(MyLogicalRepWorker->subid, + LOGICAL_REP_MSG_COMMIT_PREPARED, + get_apply_xact_size(), + prepare_data.gid); + pgstat_report_activity(STATE_IDLE, NULL); reset_apply_error_context_info(); } @@ -1014,6 +1050,13 @@ apply_handle_rollback_prepared(StringInfo s) /* Process any tables that are being synchronized in parallel. */ process_syncing_tables(rollback_data.rollback_end_lsn); + /* Update the transaction size and report */ + update_apply_change_size(LOGICAL_REP_MSG_ROLLBACK_PREPARED, false); + pgstat_report_subworker_twophase_xact(MyLogicalRepWorker->subid, + LOGICAL_REP_MSG_ROLLBACK_PREPARED, + get_apply_xact_size(), + rollback_data.gid); + pgstat_report_activity(STATE_IDLE, NULL); reset_apply_error_context_info(); } @@ -1066,6 +1109,15 @@ apply_handle_stream_prepare(StringInfo s) /* Process any tables that are being synchronized in parallel. */ process_syncing_tables(prepare_data.end_lsn); + /* + * Report the prepared streaming xact size to the stats collector in a + * prepared xact manner to make it survive over the restart. + */ + pgstat_report_subworker_twophase_xact(MyLogicalRepWorker->subid, + LOGICAL_REP_MSG_STREAM_PREPARE, + get_apply_xact_size(), + prepare_data.gid); + pgstat_report_activity(STATE_IDLE, NULL); reset_apply_error_context_info(); @@ -1438,6 +1490,12 @@ apply_handle_stream_commit(StringInfo s) /* Process any tables that are being synchronized in parallel. */ process_syncing_tables(commit_data.end_lsn); + /* Report and clean up the xid */ + pgstat_report_subworker_xact_end(MyLogicalRepWorker->subid, + InvalidOid, + LOGICAL_REP_MSG_STREAM_COMMIT, + get_apply_xact_size()); + pgstat_report_activity(STATE_IDLE, NULL); reset_apply_error_context_info(); @@ -1579,6 +1637,10 @@ apply_handle_insert(StringInfo s) slot_fill_defaults(rel, estate, remoteslot); MemoryContextSwitchTo(oldctx); + /* Update transaction size */ + update_apply_change_size(LOGICAL_REP_MSG_INSERT, + rel->localrel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE); + /* For a partitioned table, insert the tuple into a partition. */ if (rel->localrel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE) apply_handle_tuple_routing(edata, @@ -1736,7 +1798,11 @@ apply_handle_update(StringInfo s) has_oldtup ? &oldtup : &newtup); MemoryContextSwitchTo(oldctx); - /* For a partitioned table, apply update to correct partition. */ + /* Update transaction size */ + update_apply_change_size(LOGICAL_REP_MSG_UPDATE, + rel->localrel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE); + + /* For a partitioned table, apply update to corect partition. */ if (rel->localrel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE) apply_handle_tuple_routing(edata, remoteslot, &newtup, CMD_UPDATE); @@ -1870,6 +1936,10 @@ apply_handle_delete(StringInfo s) slot_store_data(remoteslot, rel, &oldtup); MemoryContextSwitchTo(oldctx); + /* Update transaction size */ + update_apply_change_size(LOGICAL_REP_MSG_DELETE, + rel->localrel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE); + /* For a partitioned table, apply delete to correct partition. */ if (rel->localrel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE) apply_handle_tuple_routing(edata, @@ -2421,6 +2491,81 @@ apply_dispatch(StringInfo s) } /* + * Subscriber side implementation equivalent to ReorderBufferChangeSize + * of the publisher. + * + * According to the logical replication message type, record major + * resource consumptions of this subscription for each message. + * At present, do not collect data from generic functions to keep + * code simplicity except for tuple length, since the implementation + * complexity versus benefit tradeoff should not be good. + * In terms of tuple length consumption, see logicalrep_read_tuple. + * + * Also, add multiple values at once in order to reduce the number + * of calls to this function, although the disadvantage of this way + * is we cannot get correct transaction size when we get an error. + * + * 'is_partitioned' is used to add some extra size. + */ +static void +update_apply_change_size(LogicalRepMsgType action, bool is_partitioned) +{ + int64 size = 0; + + switch (action) + { + /* No special memory consumption */ + case LOGICAL_REP_MSG_BEGIN: + case LOGICAL_REP_MSG_COMMIT: + case LOGICAL_REP_MSG_TRUNCATE: + case LOGICAL_REP_MSG_TYPE: + case LOGICAL_REP_MSG_ORIGIN: + case LOGICAL_REP_MSG_MESSAGE: + case LOGICAL_REP_MSG_BEGIN_PREPARE: + case LOGICAL_REP_MSG_RELATION: + break; + + case LOGICAL_REP_MSG_INSERT: + case LOGICAL_REP_MSG_UPDATE: + case LOGICAL_REP_MSG_DELETE: + Assert(!in_streamed_transaction); + + /* + * Compute size based on ApplyExecutionData. The size of + * LogicalRepRelMapEntry can be skipped because it is obtained + * from hash_search in logicalrep_rel_open. + */ + size += sizeof(ApplyExecutionData) + sizeof(EState) + + sizeof(ResultRelInfo) + sizeof(ResultRelInfo); + + /* Add some extra size if the target relation is partitioned. */ + if (is_partitioned) + size += sizeof(ModifyTableState) + sizeof(PartitionTupleRouting); + break; + + case LOGICAL_REP_MSG_PREPARE: + case LOGICAL_REP_MSG_COMMIT_PREPARED: + case LOGICAL_REP_MSG_ROLLBACK_PREPARED: + size += sizeof(FlushPosition); + break; + + case LOGICAL_REP_MSG_STREAM_START: + case LOGICAL_REP_MSG_STREAM_STOP: + case LOGICAL_REP_MSG_STREAM_COMMIT: + case LOGICAL_REP_MSG_STREAM_PREPARE: + case LOGICAL_REP_MSG_STREAM_ABORT: + default: + ereport(ERROR, + (errcode(ERRCODE_PROTOCOL_VIOLATION), + errmsg("invalid logical replication message type \"%c\"", action))); + } + + /* Update the total size of consumption when necessary */ + if (size != 0) + add_apply_xact_size(size); +} + +/* * Figure out which write/flush positions to report to the walsender process. * * We can't simply report back the last LSN the walsender sent us because the @@ -3294,6 +3439,7 @@ stream_write_change(char action, StringInfo s) BufFileWrite(stream_fd, &s->data[s->cursor], len); } + /* * Cleanup the memory for subxacts and reset the related variables. */ @@ -3605,6 +3751,32 @@ ApplyWorkerMain(Datum main_arg) } /* + * Get transaction size stored on apply error callback. + * This is used not only for error but also commit and + * rollback. Exported so that stats collector can utilize + * this value. + */ +int64 +get_apply_xact_size(void) +{ + return apply_error_callback_arg.bytes; +} + +/* Add size to apply transaction size */ +void +add_apply_xact_size(int64 size) +{ + apply_error_callback_arg.bytes += size; +} + +/* Reset transaction size on apply error callback */ +void +reset_apply_xact_size(void) +{ + apply_error_callback_arg.bytes = 0; +} + +/* * Is current process a logical replication worker? */ bool diff --git a/src/backend/utils/adt/pgstatfuncs.c b/src/backend/utils/adt/pgstatfuncs.c index a34c0b6..d7264fc 100644 --- a/src/backend/utils/adt/pgstatfuncs.c +++ b/src/backend/utils/adt/pgstatfuncs.c @@ -2414,7 +2414,7 @@ pg_stat_get_replication_slot(PG_FUNCTION_ARGS) Datum pg_stat_get_subscription_worker(PG_FUNCTION_ARGS) { -#define PG_STAT_GET_SUBSCRIPTION_WORKER_COLS 9 +#define PG_STAT_GET_SUBSCRIPTION_WORKER_COLS 15 Oid subid = PG_GETARG_OID(0); Oid subrelid; TupleDesc tupdesc; @@ -2441,19 +2441,31 @@ pg_stat_get_subscription_worker(PG_FUNCTION_ARGS) OIDOID, -1, 0); TupleDescInitEntry(tupdesc, (AttrNumber) 2, "subrelid", OIDOID, -1, 0); - TupleDescInitEntry(tupdesc, (AttrNumber) 3, "last_error_relid", + TupleDescInitEntry(tupdesc, (AttrNumber) 3, "commit_count", + INT8OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) 4, "commit_bytes", + INT8OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) 5, "error_count", + INT8OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) 6, "error_bytes", + INT8OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) 7, "abort_count", + INT8OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) 8, "abort_bytes", + INT8OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) 9, "last_error_relid", OIDOID, -1, 0); - TupleDescInitEntry(tupdesc, (AttrNumber) 4, "last_error_command", + TupleDescInitEntry(tupdesc, (AttrNumber) 10, "last_error_command", TEXTOID, -1, 0); - TupleDescInitEntry(tupdesc, (AttrNumber) 5, "last_error_xid", + TupleDescInitEntry(tupdesc, (AttrNumber) 11, "last_error_xid", XIDOID, -1, 0); - TupleDescInitEntry(tupdesc, (AttrNumber) 6, "last_error_count", + TupleDescInitEntry(tupdesc, (AttrNumber) 12, "last_error_count", INT8OID, -1, 0); - TupleDescInitEntry(tupdesc, (AttrNumber) 7, "last_error_message", + TupleDescInitEntry(tupdesc, (AttrNumber) 13, "last_error_message", TEXTOID, -1, 0); - TupleDescInitEntry(tupdesc, (AttrNumber) 8, "first_error_time", + TupleDescInitEntry(tupdesc, (AttrNumber) 14, "first_error_time", TIMESTAMPTZOID, -1, 0); - TupleDescInitEntry(tupdesc, (AttrNumber) 9, "last_error_time", + TupleDescInitEntry(tupdesc, (AttrNumber) 15, "last_error_time", TIMESTAMPTZOID, -1, 0); BlessTupleDesc(tupdesc); @@ -2471,6 +2483,14 @@ pg_stat_get_subscription_worker(PG_FUNCTION_ARGS) else nulls[i++] = true; + /* transaction stats */ + values[i++] = Int64GetDatum(wentry->xact_commit_count); + values[i++] = Int64GetDatum(wentry->xact_commit_bytes); + values[i++] = Int64GetDatum(wentry->xact_error_count); + values[i++] = Int64GetDatum(wentry->xact_error_bytes); + values[i++] = Int64GetDatum(wentry->xact_abort_count); + values[i++] = Int64GetDatum(wentry->xact_abort_bytes); + /* last_error_relid */ if (OidIsValid(wentry->relid)) values[i++] = ObjectIdGetDatum(wentry->relid); diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index 25f685f..fc654b4 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -5389,9 +5389,9 @@ proname => 'pg_stat_get_subscription_worker', prorows => '1', proisstrict => 'f', proretset => 't', provolatile => 's', proparallel => 'r', prorettype => 'record', proargtypes => 'oid oid', - proallargtypes => '{oid,oid,oid,oid,oid,text,xid,int8,text,timestamptz,timestamptz}', - proargmodes => '{i,i,o,o,o,o,o,o,o,o,o}', - proargnames => '{subid,subrelid,subid,subrelid,last_error_relid,last_error_command,last_error_xid,last_error_count,last_error_message,first_error_time,last_error_time}', + proallargtypes => '{oid,oid,oid,oid,int8,int8,int8,int8,int8,int8,oid,text,xid,int8,text,timestamptz,timestamptz}', + proargmodes => '{i,i,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o}', + proargnames => '{subid,subrelid,subid,subrelid,commit_count,commit_bytes,error_count,error_bytes,abort_count,abort_bytes,last_error_relid,last_error_command,last_error_xid,last_error_count,last_error_message,first_error_time,last_error_time}', prosrc => 'pg_stat_get_subscription_worker' }, { oid => '6118', descr => 'statistics: information about subscription', proname => 'pg_stat_get_subscription', prorows => '10', proisstrict => 'f', diff --git a/src/include/pgstat.h b/src/include/pgstat.h index 2c26b1c..266711b 100644 --- a/src/include/pgstat.h +++ b/src/include/pgstat.h @@ -86,6 +86,8 @@ typedef enum StatMsgType PGSTAT_MTYPE_DISCONNECT, PGSTAT_MTYPE_SUBSCRIPTIONPURGE, PGSTAT_MTYPE_SUBWORKERERROR, + PGSTAT_MTYPE_SUBWORKERXACTEND, + PGSTAT_MTYPE_SUBWORKERTWOPHASEXACT, } StatMsgType; /* ---------- @@ -558,6 +560,56 @@ typedef struct PgStat_MsgSubscriptionPurge } PgStat_MsgSubscriptionPurge; /* ---------- + * PgStat_MsgSubscriptionXactEnd Sent by the apply worker or the table sync worker + * to report successful transaction ends. + * ---------- + */ +typedef struct PgStat_MsgSubWorkerXactEnd +{ + PgStat_MsgHdr m_hdr; + + /* determine the worker entry */ + Oid m_databaseid; + Oid m_subid; + Oid m_subrelid; + + /* + * distinguish between transaction commits and streaming transaction + * aborts that are handled without error. + */ + LogicalRepMsgType m_command; + + /* memory consumption used by transaction */ + PgStat_Counter m_xact_bytes; + +} PgStat_MsgSubWorkerXactEnd; + +/* ---------- + * PgStat_MsgSubWorkerTwophaseXact Sent by the apply worker to make size of prepared + * txn persistent over the server restart and make it + * visible after commit prepare or rollback prepared. + * This is separated from PgStat_MsgSubWorkerXactEnd + * so that we can reduce message size of gid for other + * operations (e.g. normal COMMIT) that should happen more + * frequently than prepare operation usually. + * ---------- + */ +typedef struct PgStat_MsgSubWorkerTwophaseXact +{ + PgStat_MsgHdr m_hdr; + + /* determine the subscription */ + Oid m_databaseid; + Oid m_subid; + + LogicalRepMsgType m_command; + char m_gid[GIDSIZE]; + int gid_len; + PgStat_Counter m_xact_bytes; + +} PgStat_MsgSubWorkerTwophaseXact; + +/* ---------- * PgStat_MsgSubWorkerError Sent by the apply worker or the table sync worker to * report the error occurred during logical replication. * ---------- @@ -577,6 +629,12 @@ typedef struct PgStat_MsgSubWorkerError Oid m_subrelid; /* + * Transaction stats of subscription needs to be updated when an error + * occurs. + */ + PgStat_Counter m_xact_error_bytes; + + /* * Oid of the table that the reporter was actually processing. m_relid can * be InvalidOid if an error occurred during worker applying a * non-data-modification message such as RELATION. @@ -768,6 +826,8 @@ typedef union PgStat_Msg PgStat_MsgDisconnect msg_disconnect; PgStat_MsgSubscriptionPurge msg_subscriptionpurge; PgStat_MsgSubWorkerError msg_subworkererror; + PgStat_MsgSubWorkerXactEnd msg_subworkerxactend; + PgStat_MsgSubWorkerTwophaseXact msg_subworkertwophasexact; } PgStat_Msg; @@ -822,16 +882,23 @@ typedef struct PgStat_StatDBEntry TimestampTz stats_timestamp; /* time of db stats file update */ /* - * tables, functions, and subscription workers must be last in the struct, - * because we don't write the pointers out to the stats file. + * tables, functions, subscription workers and its prepared transaction + * stats must be last in the struct, because we don't write the pointers + * out to the stats file. * * subworker is the hash table of PgStat_StatSubWorkerEntry which stores * statistics of logical replication workers: apply worker and table sync * worker. + * + * subworkers_preparedsizes should give appropriate transaction sizes at + * either commit prepared or rollback prepared time, even when it's after + * the server restart. We have the apply worker send those statistics to + * the stats collector at prepare time. */ HTAB *tables; HTAB *functions; HTAB *subworkers; + HTAB *subworkers_preparedsizes; } PgStat_StatDBEntry; @@ -1005,6 +1072,16 @@ typedef struct PgStat_StatSubWorkerEntry PgStat_StatSubWorkerKey key; /* hash key (must be first) */ /* + * Cumulative transaction statistics of subscription worker + */ + PgStat_Counter xact_commit_count; + PgStat_Counter xact_commit_bytes; + PgStat_Counter xact_error_count; + PgStat_Counter xact_error_bytes; + PgStat_Counter xact_abort_count; + PgStat_Counter xact_abort_bytes; + + /* * Subscription worker error statistics representing an error that * occurred during application of logical replication or the initial table * synchronization. @@ -1018,6 +1095,22 @@ typedef struct PgStat_StatSubWorkerEntry char error_message[PGSTAT_SUBWORKERERROR_MSGLEN]; } PgStat_StatSubWorkerEntry; +/* prepared transaction */ +typedef struct PgStat_SW_PreparedXactKey +{ + Oid subid; + char gid[GIDSIZE]; +} PgStat_SW_PreparedXactKey; + +typedef struct PgStat_SW_PreparedXactEntry +{ + PgStat_SW_PreparedXactKey key; /* hash key */ + + Oid subid; + char gid[GIDSIZE]; + PgStat_Counter xact_size; +} PgStat_SW_PreparedXactEntry; + /* * Working state needed to accumulate per-function-call timing statistics. */ @@ -1128,6 +1221,10 @@ extern void pgstat_report_checksum_failure(void); extern void pgstat_report_replslot(const PgStat_StatReplSlotEntry *repSlotStat); extern void pgstat_report_replslot_create(const char *slotname); extern void pgstat_report_replslot_drop(const char *slotname); +extern void pgstat_report_subworker_xact_end(Oid subid, Oid subrel, + LogicalRepMsgType command, PgStat_Counter xact_size); +extern void pgstat_report_subworker_twophase_xact(Oid subid, LogicalRepMsgType command, + PgStat_Counter xact_size, char *gid); extern void pgstat_report_subworker_error(Oid subid, Oid subrelid, Oid relid, LogicalRepMsgType command, TransactionId xid, const char *errmsg); diff --git a/src/include/replication/logicalworker.h b/src/include/replication/logicalworker.h index 2ad61a0..435884d 100644 --- a/src/include/replication/logicalworker.h +++ b/src/include/replication/logicalworker.h @@ -16,4 +16,9 @@ extern void ApplyWorkerMain(Datum main_arg); extern bool IsLogicalWorker(void); +/* for transaction stats */ +extern int64 get_apply_xact_size(void); +extern void add_apply_xact_size(int64 size); +extern void reset_apply_xact_size(void); + #endif /* LOGICALWORKER_H */ diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out index d60c5a5..bd3e221 100644 --- a/src/test/regress/expected/rules.out +++ b/src/test/regress/expected/rules.out @@ -2097,6 +2097,12 @@ pg_stat_subscription| SELECT su.oid AS subid, pg_stat_subscription_workers| SELECT w.subid, s.subname, w.subrelid, + w.commit_count, + w.commit_bytes, + w.error_count, + w.error_bytes, + w.abort_count, + w.abort_bytes, w.last_error_relid, w.last_error_command, w.last_error_xid, @@ -2112,7 +2118,7 @@ pg_stat_subscription_workers| SELECT w.subid, pg_subscription_rel.srrelid AS relid FROM pg_subscription_rel WHERE (pg_subscription_rel.srsubstate <> 'r'::"char")) sr, - (LATERAL pg_stat_get_subscription_worker(sr.subid, sr.relid) w(subid, subrelid, last_error_relid, last_error_command, last_error_xid, last_error_count, last_error_message, first_error_time, last_error_time) + (LATERAL pg_stat_get_subscription_worker(sr.subid, sr.relid) w(subid, subrelid, commit_count, commit_bytes, error_count, error_bytes, abort_count, abort_bytes, last_error_relid, last_error_command, last_error_xid, last_error_count, last_error_message, first_error_time, last_error_time) JOIN pg_subscription s ON ((w.subid = s.oid))); pg_stat_sys_indexes| SELECT pg_stat_all_indexes.relid, pg_stat_all_indexes.indexrelid, diff --git a/src/test/subscription/t/027_worker_xact_stats.pl b/src/test/subscription/t/027_worker_xact_stats.pl new file mode 100644 index 0000000..e09be42 --- /dev/null +++ b/src/test/subscription/t/027_worker_xact_stats.pl @@ -0,0 +1,146 @@ + +# Copyright (c) 2021, PostgreSQL Global Development Group + +# Tests for subscription worker statistics during apply. +use strict; +use warnings; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More tests => 2; + +# Create publisher node +my $node_publisher = PostgreSQL::Test::Cluster->new('publisher'); +$node_publisher->init(allows_streaming => 'logical'); +$node_publisher->append_conf( + 'postgresql.conf', qq[ +logical_decoding_work_mem = 64kB +max_prepared_transactions = 10 +max_wal_senders = 10 +wal_sender_timeout = 0 +]); +$node_publisher->start; + +# Create subscriber node +my $node_subscriber = PostgreSQL::Test::Cluster->new('subscriber'); +$node_subscriber->init(allows_streaming => 'logical'); +$node_subscriber->append_conf( + 'postgresql.conf', qq[ +max_prepared_transactions = 10 +log_min_messages = DEBUG1 +]); +$node_subscriber->start; + +# Setup structures on the publisher and the subscriber +$node_publisher->safe_psql('postgres', "CREATE TABLE test_tab (a int)"); +$node_subscriber->safe_psql('postgres', + "CREATE TABLE test_tab (a int primary key);"); + +# Setup logical replication +my $publisher_connstr = $node_publisher->connstr . ' dbname=postgres'; +$node_publisher->safe_psql('postgres', + "CREATE PUBLICATION tap_pub FOR TABLE test_tab"); + +my $appname = 'tap_sub'; +my $subopts = 'streaming = on, two_phase = on, copy_data = false'; +$node_subscriber->safe_psql( + 'postgres', qq[ +CREATE SUBSCRIPTION tap_sub +CONNECTION '$publisher_connstr application_name=$appname' +PUBLICATION tap_pub WITH ($subopts); +]); + +$node_publisher->wait_for_catchup($appname); + +# There's no entry at the beginning +my $result = $node_subscriber->safe_psql('postgres', + "SELECT count(*) FROM pg_stat_subscription_workers;"); +is($result, q(0), 'no entry for transaction stats yet'); + +# COMMIT +$node_publisher->safe_psql('postgres', + "BEGIN; INSERT INTO test_tab VALUES (1); COMMIT;"); + +$node_subscriber->poll_query_until('postgres', + "SELECT count(1) = 1 FROM pg_stat_subscription_workers where commit_count = 1;" +) or die "didn't get updates of xact stats by commit"; + +# Now, stats collector make the bytes updated also. +$result = $node_subscriber->safe_psql('postgres', + "SELECT commit_bytes > 0 FROM pg_stat_subscription_workers;"); +is($result, q(t), 'got consumed bytes'); + +# STREAM COMMIT +$node_publisher->safe_psql('postgres', + "BEGIN; INSERT INTO test_tab VALUES(generate_series(1001, 2000)); COMMIT;" +); +$node_subscriber->poll_query_until('postgres', + "SELECT count(1) = 1 FROM pg_stat_subscription_workers where commit_count = 2;" +) or die "didn't get updates of xact stats by stream commit"; + +# STREAM PREPARE & COMMIT PREPARED +# This should restore the xact size before the shutdown. +$node_publisher->safe_psql( + 'postgres', q[ +BEGIN; +INSERT INTO test_tab VALUES(generate_series(2001, 3000)); +PREPARE TRANSACTION 'gid1'; +]); + +# This streamed prepare is not displayed until the commit prepared +# or rollback prepared. Hence, there's no way to confirm that +# stats collector has received the bytes of prepared transaction. +# So, instead of checking the view, issue one more committed transaction +# after the prepare and make sure that this commit's update is done, +# which should mean the previous streamed prepare is already processed +# by the stats collector as well. +$node_publisher->safe_psql('postgres', + "BEGIN; INSERT INTO test_tab VALUES (2); COMMIT;"); +$node_subscriber->poll_query_until('postgres', + "SELECT count(1) = 1 FROM pg_stat_subscription_workers where commit_count = 3;" +) or die "didn't process the updates of committed transaction"; + +$node_subscriber->restart; + +# Get commit_bytes before commit prepared. +my $tmp = $node_subscriber->safe_psql('postgres', + "SELECT commit_bytes FROM pg_stat_subscription_workers where commit_count = 3;" +); + +# Commit prepared increments the commit_count. +$node_publisher->safe_psql('postgres', "COMMIT PREPARED 'gid1'"); +$node_subscriber->poll_query_until('postgres', + "SELECT count(1) = 1 FROM pg_stat_subscription_workers where commit_count = 4;" + ) + or die + "didn't get updates of xact stats by stream prepare and commit prepared"; + +$node_subscriber->poll_query_until('postgres', + "SELECT commit_bytes > $tmp FROM pg_stat_subscription_workers where commit_count = 4;" +); + +# ROLLBACK PREPARED +$node_publisher->safe_psql( + 'postgres', q[ +BEGIN; +INSERT INTO test_tab VALUES (3); +PREPARE TRANSACTION 'gid2'; +]); +$node_publisher->safe_psql('postgres', "ROLLBACK PREPARED 'gid2'"); +$node_subscriber->poll_query_until('postgres', + "SELECT count(1) = 1 FROM pg_stat_subscription_workers where abort_count = 1;" +) or die "didn't get updates of xact stats by rollback prepared"; + +# error stats (by duplication error) +$node_publisher->safe_psql( + 'postgres', q[ +BEGIN; +INSERT INTO test_tab VALUES (1); +COMMIT; +]); + +$node_subscriber->poll_query_until('postgres', + "SELECT count(1) = 1 FROM pg_stat_subscription_workers where error_count > 0;" +) or die "didn't get updates of xact stats by error"; + +$node_subscriber->stop('fast'); +$node_publisher->stop('fast'); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f41ef0d..3403667 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -1945,6 +1945,8 @@ PgStat_MsgResetslrucounter PgStat_MsgSLRU PgStat_MsgSubscriptionPurge PgStat_MsgSubWorkerError +PgStat_MsgSubWorkerTwophaseXact +PgStat_MsgSubWorkerXactEnd PgStat_MsgTabpurge PgStat_MsgTabstat PgStat_MsgTempFile @@ -1958,6 +1960,8 @@ PgStat_StatFuncEntry PgStat_StatReplSlotEntry PgStat_StatSubWorkerEntry PgStat_StatSubWorkerKey +PgStat_SW_PreparedXactKey +PgStat_SW_PreparedXactEntry PgStat_StatTabEntry PgStat_SubXactStatus PgStat_TableCounts -- 2.2.0 ^ permalink raw reply [nested|flat] 113+ messages in thread
* Re: Failed transaction statistics to measure the logical replication progress @ 2021-11-17 03:18 Masahiko Sawada <[email protected]> parent: [email protected] <[email protected]> 1 sibling, 1 reply; 113+ messages in thread From: Masahiko Sawada @ 2021-11-17 03:18 UTC (permalink / raw) To: [email protected] <[email protected]>; +Cc: vignesh C <[email protected]>; Greg Nancarrow <[email protected]>; Amit Kapila <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]> On Tue, Nov 16, 2021 at 9:34 PM [email protected] <[email protected]> wrote: > > On Monday, November 15, 2021 9:14 PM I wrote: > > I've conducted some update for this. > > (The rebased part is only C code and checked by pgindent) > I'll update my patches since a new skip xid patch > has been shared in [1]. > > This version includes some minor renames of functions > that are related to transaction sizes. I've looked at v12-0001 patch. Here are some comments: - TupleDescInitEntry(tupdesc, (AttrNumber) 3, "relid", + TupleDescInitEntry(tupdesc, (AttrNumber) 3, "last_error_relid", OIDOID, -1, 0); - TupleDescInitEntry(tupdesc, (AttrNumber) 4, "command", + TupleDescInitEntry(tupdesc, (AttrNumber) 4, "last_error_command", TEXTOID, -1, 0); - TupleDescInitEntry(tupdesc, (AttrNumber) 5, "xid", + TupleDescInitEntry(tupdesc, (AttrNumber) 5, "last_error_xid", XIDOID, -1, 0); - TupleDescInitEntry(tupdesc, (AttrNumber) 6, "error_count", + TupleDescInitEntry(tupdesc, (AttrNumber) 6, "last_error_count", INT8OID, -1, 0); - TupleDescInitEntry(tupdesc, (AttrNumber) 7, "error_message", + TupleDescInitEntry(tupdesc, (AttrNumber) 7, "last_error_message", If renaming column names clarifies those meanings, the above changes should be included into my patch that introduces pg_stat_subscription_workers view? --- I think that exporting PartitionTupleRouting should not be done in the one patch together with renaming the view columns. There is not relevance between them at all. If it's used by v12-0002 patch, I think it should be included in that patch or in another separate patch. Regards, -- Masahiko Sawada EDB: https://www.enterprisedb.com/ ^ permalink raw reply [nested|flat] 113+ messages in thread
* RE: Failed transaction statistics to measure the logical replication progress @ 2021-11-17 04:14 [email protected] <[email protected]> parent: Masahiko Sawada <[email protected]> 0 siblings, 1 reply; 113+ messages in thread From: [email protected] @ 2021-11-17 04:14 UTC (permalink / raw) To: 'Masahiko Sawada' <[email protected]>; +Cc: vignesh C <[email protected]>; Greg Nancarrow <[email protected]>; Amit Kapila <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]> On Wednesday, November 17, 2021 12:19 PM Masahiko Sawada <[email protected]> wrote: > On Tue, Nov 16, 2021 at 9:34 PM [email protected] > <[email protected]> wrote: > > > > On Monday, November 15, 2021 9:14 PM I wrote: > > > I've conducted some update for this. > > > (The rebased part is only C code and checked by pgindent) > > I'll update my patches since a new skip xid patch has been shared in > > [1]. > > > > This version includes some minor renames of functions that are related > > to transaction sizes. > > I've looked at v12-0001 patch. Here are some comments: Thank you for paying attention to this thread ! > - TupleDescInitEntry(tupdesc, (AttrNumber) 3, "relid", > + TupleDescInitEntry(tupdesc, (AttrNumber) 3, "last_error_relid", > OIDOID, -1, 0); > - TupleDescInitEntry(tupdesc, (AttrNumber) 4, "command", > + TupleDescInitEntry(tupdesc, (AttrNumber) 4, "last_error_command", > TEXTOID, -1, 0); > - TupleDescInitEntry(tupdesc, (AttrNumber) 5, "xid", > + TupleDescInitEntry(tupdesc, (AttrNumber) 5, "last_error_xid", > XIDOID, -1, 0); > - TupleDescInitEntry(tupdesc, (AttrNumber) 6, "error_count", > + TupleDescInitEntry(tupdesc, (AttrNumber) 6, "last_error_count", > INT8OID, -1, 0); > - TupleDescInitEntry(tupdesc, (AttrNumber) 7, "error_message", > + TupleDescInitEntry(tupdesc, (AttrNumber) 7, "last_error_message", > > If renaming column names clarifies those meanings, the above changes should > be included into my patch that introduces pg_stat_subscription_workers > view? At first, your column names of pg_stat_subscription_workers look totally OK to me by itself and I thought I should take care of those renaming at the commit timing of my stats patches. But, if you agree with the new names above and fixing your patch doesn't bother you, I'd appreciate your help ! > I think that exporting PartitionTupleRouting should not be done in the one > patch together with renaming the view columns. There is not relevance > between them at all. If it's used by v12-0002 patch, I think it should be included > in that patch or in another separate patch. Yes, it's used by v12-0002. Absolutely you are right. When you update the patch like above, I would like to make it independent. Best Regards, Takamichi Osumi ^ permalink raw reply [nested|flat] 113+ messages in thread
* Re: Failed transaction statistics to measure the logical replication progress @ 2021-11-17 13:00 Amit Kapila <[email protected]> parent: [email protected] <[email protected]> 0 siblings, 1 reply; 113+ messages in thread From: Amit Kapila @ 2021-11-17 13:00 UTC (permalink / raw) To: [email protected] <[email protected]>; +Cc: Masahiko Sawada <[email protected]>; vignesh C <[email protected]>; Greg Nancarrow <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]> On Wed, Nov 17, 2021 at 9:44 AM [email protected] <[email protected]> wrote: > > On Wednesday, November 17, 2021 12:19 PM Masahiko Sawada <[email protected]> wrote: > > On Tue, Nov 16, 2021 at 9:34 PM [email protected] > > <[email protected]> wrote: > > > > > > On Monday, November 15, 2021 9:14 PM I wrote: > > > > I've conducted some update for this. > > > > (The rebased part is only C code and checked by pgindent) > > > I'll update my patches since a new skip xid patch has been shared in > > > [1]. > > > > > > This version includes some minor renames of functions that are related > > > to transaction sizes. > > > > I've looked at v12-0001 patch. Here are some comments: > Thank you for paying attention to this thread ! > > > > - TupleDescInitEntry(tupdesc, (AttrNumber) 3, "relid", > > + TupleDescInitEntry(tupdesc, (AttrNumber) 3, "last_error_relid", > > OIDOID, -1, 0); > > - TupleDescInitEntry(tupdesc, (AttrNumber) 4, "command", > > + TupleDescInitEntry(tupdesc, (AttrNumber) 4, "last_error_command", > > TEXTOID, -1, 0); > > - TupleDescInitEntry(tupdesc, (AttrNumber) 5, "xid", > > + TupleDescInitEntry(tupdesc, (AttrNumber) 5, "last_error_xid", > > XIDOID, -1, 0); > > - TupleDescInitEntry(tupdesc, (AttrNumber) 6, "error_count", > > + TupleDescInitEntry(tupdesc, (AttrNumber) 6, "last_error_count", > > INT8OID, -1, 0); > > - TupleDescInitEntry(tupdesc, (AttrNumber) 7, "error_message", > > + TupleDescInitEntry(tupdesc, (AttrNumber) 7, "last_error_message", > > > > If renaming column names clarifies those meanings, the above changes should > > be included into my patch that introduces pg_stat_subscription_workers > > view? Right. > At first, your column names of pg_stat_subscription_workers look totally OK to me by itself > and I thought I should take care of those renaming at the commit timing of my stats patches. > Can you please tell us why you think the names in your proposed patch are better than the existing names proposed in Sawada-San's patch? Is it because those fields always contain the information of the last or latest error that occurred in the corresponding subscription worker? If so, I am not very sure if that is a good reason to increase the length of most of the column names but if you and others feel that is helpful then it is better to do it as part of Sawada-San's patch. -- With Regards, Amit Kapila. ^ permalink raw reply [nested|flat] 113+ messages in thread
* RE: Failed transaction statistics to measure the logical replication progress @ 2021-11-17 13:42 [email protected] <[email protected]> parent: Amit Kapila <[email protected]> 0 siblings, 1 reply; 113+ messages in thread From: [email protected] @ 2021-11-17 13:42 UTC (permalink / raw) To: 'Amit Kapila' <[email protected]>; +Cc: Masahiko Sawada <[email protected]>; vignesh C <[email protected]>; Greg Nancarrow <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]> On Wednesday, November 17, 2021 10:00 PM Amit Kapila <[email protected]> wrote: > On Wed, Nov 17, 2021 at 9:44 AM [email protected] > <[email protected]> wrote: > > > > On Wednesday, November 17, 2021 12:19 PM Masahiko Sawada > <[email protected]> wrote: > > > On Tue, Nov 16, 2021 at 9:34 PM [email protected] > > > <[email protected]> wrote: > > > > > > > > On Monday, November 15, 2021 9:14 PM I wrote: > > > > > I've conducted some update for this. > > > > > (The rebased part is only C code and checked by pgindent) > > > > I'll update my patches since a new skip xid patch has been shared > > > > in [1]. > > > > > > > > This version includes some minor renames of functions that are > > > > related to transaction sizes. > > > > > > I've looked at v12-0001 patch. Here are some comments: > > Thank you for paying attention to this thread ! > > > > > > > - TupleDescInitEntry(tupdesc, (AttrNumber) 3, "relid", > > > + TupleDescInitEntry(tupdesc, (AttrNumber) 3, "last_error_relid", > > > OIDOID, -1, 0); > > > - TupleDescInitEntry(tupdesc, (AttrNumber) 4, "command", > > > + TupleDescInitEntry(tupdesc, (AttrNumber) 4, > > > + "last_error_command", > > > TEXTOID, -1, 0); > > > - TupleDescInitEntry(tupdesc, (AttrNumber) 5, "xid", > > > + TupleDescInitEntry(tupdesc, (AttrNumber) 5, "last_error_xid", > > > XIDOID, -1, 0); > > > - TupleDescInitEntry(tupdesc, (AttrNumber) 6, "error_count", > > > + TupleDescInitEntry(tupdesc, (AttrNumber) 6, "last_error_count", > > > INT8OID, -1, 0); > > > - TupleDescInitEntry(tupdesc, (AttrNumber) 7, "error_message", > > > + TupleDescInitEntry(tupdesc, (AttrNumber) 7, > > > + "last_error_message", > > > > > > If renaming column names clarifies those meanings, the above changes > > > should be included into my patch that introduces > > > pg_stat_subscription_workers view? > > Right. > > > At first, your column names of pg_stat_subscription_workers look > > totally OK to me by itself and I thought I should take care of those renaming at > the commit timing of my stats patches. > > > > Can you please tell us why you think the names in your proposed patch are > better than the existing names proposed in Sawada-San's patch? Is it because > those fields always contain the information of the last or latest error that > occurred in the corresponding subscription worker? This is one reason. Another big reason comes from the final alignment when we list up all columns of both patches. The patches in this thread is trying to introduce a column that indicates cumulative count of error to show all error counts that the worker got in the past. In this thread, after two or three improvements, this column name has reached to a simple one 'error_count' (aligned with 'commit_count' and 'abort_count'). Then we need to differentiate what this thread's patch is trying to introduce and what skip xid patch is introducing. > If so, I am not very sure if that is a good reason to increase the length of most of > the column names but if you and others feel that is helpful then it is better to do > it as part of Sawada-San's patch. Yes. On this point, it was a mistake that I handled that changes. It'd be better that Sawada-san takes care of them once the names have been fixed. Best Regards, Takamichi Osumi ^ permalink raw reply [nested|flat] 113+ messages in thread
* Re: Failed transaction statistics to measure the logical replication progress @ 2021-11-18 03:26 Amit Kapila <[email protected]> parent: [email protected] <[email protected]> 0 siblings, 2 replies; 113+ messages in thread From: Amit Kapila @ 2021-11-18 03:26 UTC (permalink / raw) To: [email protected] <[email protected]>; +Cc: Masahiko Sawada <[email protected]>; vignesh C <[email protected]>; Greg Nancarrow <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]> On Wed, Nov 17, 2021 at 7:12 PM [email protected] <[email protected]> wrote: > > On Wednesday, November 17, 2021 10:00 PM Amit Kapila <[email protected]> wrote: > > > > Can you please tell us why you think the names in your proposed patch are > > better than the existing names proposed in Sawada-San's patch? Is it because > > those fields always contain the information of the last or latest error that > > occurred in the corresponding subscription worker? > This is one reason. > > Another big reason comes from the final alignment when we list up all columns of both patches. > The patches in this thread is trying to introduce a column that indicates > cumulative count of error to show all error counts that the worker got in the past. > Okay, I see your point and it makes sense to rename columns after these other stats. I am not able to come up with any better names than what is being used here. Sawada-San, do you agree with this, or do let us know if you have any better ideas? BTW, I think the way you are computing error_count in pgstat_recv_subworker_error() doesn't seem correct to me because it will accumulate the counter/bytes for the same error again and again. You might want to update these counters after we have checked that the received error is not the same as the previous one. -- With Regards, Amit Kapila. ^ permalink raw reply [nested|flat] 113+ messages in thread
* Re: Failed transaction statistics to measure the logical replication progress @ 2021-11-18 11:34 vignesh C <[email protected]> parent: [email protected] <[email protected]> 1 sibling, 1 reply; 113+ messages in thread From: vignesh C @ 2021-11-18 11:34 UTC (permalink / raw) To: [email protected] <[email protected]>; +Cc: Greg Nancarrow <[email protected]>; Masahiko Sawada <[email protected]>; Amit Kapila <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]> On Tue, Nov 16, 2021 at 6:04 PM [email protected] <[email protected]> wrote: > > On Monday, November 15, 2021 9:14 PM I wrote: > > I've conducted some update for this. > > (The rebased part is only C code and checked by pgindent) > I'll update my patches since a new skip xid patch > has been shared in [1]. > > This version includes some minor renames of functions > that are related to transaction sizes. Thanks for the updated patch, Few comments: 1) since pgstat_get_subworker_prepared_txn is called from only one place and create is passed as true, we can remove create function parameter or the function could be removed. + * Return subscription worker entry with the given subscription OID and + * gid. + * ---------- + */ +static PgStat_SW_PreparedXactEntry * +pgstat_get_subworker_prepared_txn(Oid databaseid, Oid subid, + char *gid, bool create) +{ + PgStat_StatDBEntry *dbentry; + PgStat_SW_PreparedXactKey key; 2) Include subworker prepared transactions also /* * Don't create tables/functions/subworkers hashtables for * uninteresting databases. */ if (onlydb != InvalidOid) { if (dbbuf.databaseid != onlydb && dbbuf.databaseid != InvalidOid) break; } 3) Similarly it should be mentioned in: reset_dbentry_counters function header, pgstat_read_db_statsfile function header and pgstat_get_db_entry function comments. 4) I felt we can remove "COMMIT of streaming transaction", since only commit and commit prepared are the user operations. Shall we change it to "COMMIT and COMMIT PREPARED will increment this counter." + <structfield>commit_count</structfield> <type>bigint</type> + </para> + <para> + Number of transactions successfully applied in this subscription. + COMMIT, COMMIT of streaming transaction and COMMIT PREPARED increments + this counter. + </para></entry> + </row> 5) PgStat_SW_PreparedXactEntry should be before PgStat_SW_PreparedXactKey PgStat_StatSubWorkerEntry PgStat_StatSubWorkerKey +PgStat_SW_PreparedXactKey +PgStat_SW_PreparedXactEntry PgStat_StatTabEntry PgStat_SubXactStatus 6) This change is not required @@ -293,6 +306,7 @@ static inline void cleanup_subxact_info(void); static void stream_cleanup_files(Oid subid, TransactionId xid); static void stream_open_file(Oid subid, TransactionId xid, bool first); static void stream_write_change(char action, StringInfo s); + static void stream_close_file(void); Regards, Vignesh ^ permalink raw reply [nested|flat] 113+ messages in thread
* RE: Failed transaction statistics to measure the logical replication progress @ 2021-11-18 14:39 [email protected] <[email protected]> parent: vignesh C <[email protected]> 0 siblings, 0 replies; 113+ messages in thread From: [email protected] @ 2021-11-18 14:39 UTC (permalink / raw) To: 'vignesh C' <[email protected]>; +Cc: Greg Nancarrow <[email protected]>; Masahiko Sawada <[email protected]>; Amit Kapila <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]> On Thursday, November 18, 2021 8:35 PM vignesh C <[email protected]> wrote: > On Tue, Nov 16, 2021 at 6:04 PM [email protected] > <[email protected]> wrote: > > > > On Monday, November 15, 2021 9:14 PM I wrote: > > > I've conducted some update for this. > > > (The rebased part is only C code and checked by pgindent) > > I'll update my patches since a new skip xid patch has been shared in > > [1]. > > > > This version includes some minor renames of functions that are related > > to transaction sizes. > > Thanks for the updated patch, Few comments: Thank you for checking the patches ! > 1) since pgstat_get_subworker_prepared_txn is called from only one place and > create is passed as true, we can remove create function parameter or the > function could be removed. > + * Return subscription worker entry with the given subscription OID and > + * gid. > + * ---------- > + */ > +static PgStat_SW_PreparedXactEntry * > +pgstat_get_subworker_prepared_txn(Oid databaseid, Oid subid, > + char > *gid, bool create) > +{ > + PgStat_StatDBEntry *dbentry; > + PgStat_SW_PreparedXactKey key; Removed the parameter. > 2) Include subworker prepared transactions also > /* > * Don't create tables/functions/subworkers hashtables for > * uninteresting databases. > */ > if (onlydb != InvalidOid) > { > if (dbbuf.databaseid != onlydb && > dbbuf.databaseid != InvalidOid) > break; > } Fixed. > 3) Similarly it should be mentioned in: > reset_dbentry_counters function header, pgstat_read_db_statsfile function > header and pgstat_get_db_entry function comments. Fixed. > 4) I felt we can remove "COMMIT of streaming transaction", since only commit > and commit prepared are the user operations. Shall we change it to "COMMIT > and COMMIT PREPARED will increment this counter." > + <structfield>commit_count</structfield> <type>bigint</type> > + </para> > + <para> > + Number of transactions successfully applied in this subscription. > + COMMIT, COMMIT of streaming transaction and COMMIT > PREPARED increments > + this counter. > + </para></entry> > + </row> You are right ! Fixed. > 5) PgStat_SW_PreparedXactEntry should be before > PgStat_SW_PreparedXactKey PgStat_StatSubWorkerEntry > PgStat_StatSubWorkerKey > +PgStat_SW_PreparedXactKey > +PgStat_SW_PreparedXactEntry > PgStat_StatTabEntry > PgStat_SubXactStatus Fixed. > 6) This change is not required > @@ -293,6 +306,7 @@ static inline void cleanup_subxact_info(void); static > void stream_cleanup_files(Oid subid, TransactionId xid); static void > stream_open_file(Oid subid, TransactionId xid, bool first); static void > stream_write_change(char action, StringInfo s); > + > static void stream_close_file(void); Removed. Other changes are 1. refined the commit message of v13-0003*. 2. made the additional comment for ApplyErrorCallbackArg simple. 3. wrote more explanations about update_apply_change_size() as comment. 4. changed the behavior of pgstat_recv_subworker_error so that it can store stats info only once per error. 5. added one simple test for PREPARE and COMMIT PREPARED. This used v23 skip xid patch [1]. (I will remove v13-0001* when the column names are fixed and Sawada-san starts to take care of the column name definitions) [1] - https://www.postgresql.org/message-id/CAD21AoA5jupM6O%3DpYsyfaxQ1aMX-en8%3DQNgpW6KfXsg7_CS0CQ%40mail... Best Regards, Takamichi Osumi Attachments: [application/octet-stream] v13-0001-Rename-existing-columns-of-pg_stat_subscription_.patch (9.3K, ../../TYCPR01MB8373533A5C24BDDA516DA7E1ED9B9@TYCPR01MB8373.jpnprd01.prod.outlook.com/2-v13-0001-Rename-existing-columns-of-pg_stat_subscription_.patch) download | inline diff: From c81685d93b05ceea6da04bd1f93dbc661dbeb991 Mon Sep 17 00:00:00 2001 From: Osumi Takamichi <[email protected]> Date: Thu, 18 Nov 2021 11:43:46 +0000 Subject: [PATCH v13 1/3] Rename existing columns of pg_stat_subscription_workers --- doc/src/sgml/monitoring.sgml | 24 ++++++++++++------------ src/backend/catalog/system_views.sql | 10 +++++----- src/backend/utils/adt/pgstatfuncs.c | 20 ++++++++++---------- src/include/catalog/pg_proc.dat | 2 +- src/test/regress/expected/rules.out | 12 ++++++------ src/test/subscription/t/026_error_report.pl | 8 ++++---- 6 files changed, 38 insertions(+), 38 deletions(-) diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml index daf9fe8..5b10d18 100644 --- a/doc/src/sgml/monitoring.sgml +++ b/doc/src/sgml/monitoring.sgml @@ -3103,31 +3103,31 @@ SELECT pid, wait_event_type, wait_event FROM pg_stat_activity WHERE wait_event i <row> <entry role="catalog_table_entry"><para role="column_definition"> - <structfield>relid</structfield> <type>oid</type> + <structfield>last_error_relid</structfield> <type>oid</type> </para> <para> OID of the relation that the worker was processing when the - error occurred + last error occurred </para></entry> </row> <row> <entry role="catalog_table_entry"><para role="column_definition"> - <structfield>command</structfield> <type>text</type> + <structfield>last_error_command</structfield> <type>text</type> </para> <para> - Name of command being applied when the error occurred. This field - is always NULL if the error was reported during the initial data - copy. + Name of the last command being applied when the error occurred. + This field is always NULL if the error was reported during the + initial data copy. </para></entry> </row> <row> <entry role="catalog_table_entry"><para role="column_definition"> - <structfield>xid</structfield> <type>xid</type> + <structfield>last_error_xid</structfield> <type>xid</type> </para> <para> - Transaction ID of the publisher node being applied when the error + Transaction ID of the publisher node being applied when the last error occurred. This field is always NULL if the error was reported during the initial data copy. </para></entry> @@ -3135,19 +3135,19 @@ SELECT pid, wait_event_type, wait_event FROM pg_stat_activity WHERE wait_event i <row> <entry role="catalog_table_entry"><para role="column_definition"> - <structfield>error_count</structfield> <type>uint8</type> + <structfield>last_error_count</structfield> <type>uint8</type> </para> <para> - Number of consecutive times the error occurred + Number of consecutive times the last error occurred </para></entry> </row> <row> <entry role="catalog_table_entry"><para role="column_definition"> - <structfield>error_message</structfield> <type>text</type> + <structfield>last_error_message</structfield> <type>text</type> </para> <para> - The error message + The last error message </para></entry> </row> diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index cb2f77c..ecf1a0b 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1267,11 +1267,11 @@ CREATE VIEW pg_stat_subscription_workers AS w.subid, s.subname, w.subrelid, - w.relid, - w.command, - w.xid, - w.error_count, - w.error_message, + w.last_error_relid, + w.last_error_command, + w.last_error_xid, + w.last_error_count, + w.last_error_message, w.first_error_time, w.last_error_time FROM (SELECT diff --git a/src/backend/utils/adt/pgstatfuncs.c b/src/backend/utils/adt/pgstatfuncs.c index b19729d..a34c0b6 100644 --- a/src/backend/utils/adt/pgstatfuncs.c +++ b/src/backend/utils/adt/pgstatfuncs.c @@ -2441,15 +2441,15 @@ pg_stat_get_subscription_worker(PG_FUNCTION_ARGS) OIDOID, -1, 0); TupleDescInitEntry(tupdesc, (AttrNumber) 2, "subrelid", OIDOID, -1, 0); - TupleDescInitEntry(tupdesc, (AttrNumber) 3, "relid", + TupleDescInitEntry(tupdesc, (AttrNumber) 3, "last_error_relid", OIDOID, -1, 0); - TupleDescInitEntry(tupdesc, (AttrNumber) 4, "command", + TupleDescInitEntry(tupdesc, (AttrNumber) 4, "last_error_command", TEXTOID, -1, 0); - TupleDescInitEntry(tupdesc, (AttrNumber) 5, "xid", + TupleDescInitEntry(tupdesc, (AttrNumber) 5, "last_error_xid", XIDOID, -1, 0); - TupleDescInitEntry(tupdesc, (AttrNumber) 6, "error_count", + TupleDescInitEntry(tupdesc, (AttrNumber) 6, "last_error_count", INT8OID, -1, 0); - TupleDescInitEntry(tupdesc, (AttrNumber) 7, "error_message", + TupleDescInitEntry(tupdesc, (AttrNumber) 7, "last_error_message", TEXTOID, -1, 0); TupleDescInitEntry(tupdesc, (AttrNumber) 8, "first_error_time", TIMESTAMPTZOID, -1, 0); @@ -2471,28 +2471,28 @@ pg_stat_get_subscription_worker(PG_FUNCTION_ARGS) else nulls[i++] = true; - /* relid */ + /* last_error_relid */ if (OidIsValid(wentry->relid)) values[i++] = ObjectIdGetDatum(wentry->relid); else nulls[i++] = true; - /* command */ + /* last_error_command */ if (wentry->command != 0) values[i++] = CStringGetTextDatum(logicalrep_message_type(wentry->command)); else nulls[i++] = true; - /* xid */ + /* last_error_xid */ if (TransactionIdIsValid(wentry->xid)) values[i++] = TransactionIdGetDatum(wentry->xid); else nulls[i++] = true; - /* error_count */ + /* last_error_count */ values[i++] = Int64GetDatum(wentry->error_count); - /* error_message */ + /* last_error_message */ values[i++] = CStringGetTextDatum(wentry->error_message); /* first_error_time */ diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index 50e1c7b..25f685f 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -5391,7 +5391,7 @@ prorettype => 'record', proargtypes => 'oid oid', proallargtypes => '{oid,oid,oid,oid,oid,text,xid,int8,text,timestamptz,timestamptz}', proargmodes => '{i,i,o,o,o,o,o,o,o,o,o}', - proargnames => '{subid,subrelid,subid,subrelid,relid,command,xid,error_count,error_message,first_error_time,last_error_time}', + proargnames => '{subid,subrelid,subid,subrelid,last_error_relid,last_error_command,last_error_xid,last_error_count,last_error_message,first_error_time,last_error_time}', prosrc => 'pg_stat_get_subscription_worker' }, { oid => '6118', descr => 'statistics: information about subscription', proname => 'pg_stat_get_subscription', prorows => '10', proisstrict => 'f', diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out index cb6da2c..d60c5a5 100644 --- a/src/test/regress/expected/rules.out +++ b/src/test/regress/expected/rules.out @@ -2097,11 +2097,11 @@ pg_stat_subscription| SELECT su.oid AS subid, pg_stat_subscription_workers| SELECT w.subid, s.subname, w.subrelid, - w.relid, - w.command, - w.xid, - w.error_count, - w.error_message, + w.last_error_relid, + w.last_error_command, + w.last_error_xid, + w.last_error_count, + w.last_error_message, w.first_error_time, w.last_error_time FROM ( SELECT pg_subscription.oid AS subid, @@ -2112,7 +2112,7 @@ pg_stat_subscription_workers| SELECT w.subid, pg_subscription_rel.srrelid AS relid FROM pg_subscription_rel WHERE (pg_subscription_rel.srsubstate <> 'r'::"char")) sr, - (LATERAL pg_stat_get_subscription_worker(sr.subid, sr.relid) w(subid, subrelid, relid, command, xid, error_count, error_message, first_error_time, last_error_time) + (LATERAL pg_stat_get_subscription_worker(sr.subid, sr.relid) w(subid, subrelid, last_error_relid, last_error_command, last_error_xid, last_error_count, last_error_message, first_error_time, last_error_time) JOIN pg_subscription s ON ((w.subid = s.oid))); pg_stat_sys_indexes| SELECT pg_stat_all_indexes.relid, pg_stat_all_indexes.indexrelid, diff --git a/src/test/subscription/t/026_error_report.pl b/src/test/subscription/t/026_error_report.pl index 1227654..ca30ab2 100644 --- a/src/test/subscription/t/026_error_report.pl +++ b/src/test/subscription/t/026_error_report.pl @@ -15,8 +15,8 @@ sub test_subscription_error my $check_sql = qq[ SELECT count(1) > 0 FROM pg_stat_subscription_workers -WHERE relid = '$relname'::regclass]; - $check_sql .= " AND xid = '$xid'::xid;" if $xid ne ''; +WHERE last_error_relid = '$relname'::regclass]; + $check_sql .= " AND last_error_xid = '$xid'::xid;" if $xid ne ''; # Wait for the error statistics to be updated. $node->poll_query_until( @@ -26,9 +26,9 @@ WHERE relid = '$relname'::regclass]; my $result = $node->safe_psql( 'postgres', qq[ -SELECT subname, command, relid::regclass, error_count > 0 +SELECT subname, last_error_command, last_error_relid::regclass, last_error_count > 0 FROM pg_stat_subscription_workers -WHERE relid = '$relname'::regclass; +WHERE last_error_relid = '$relname'::regclass; ]); is($result, $expected_error, $msg); } -- 2.2.0 [application/octet-stream] v13-0002-Export-PartitionTupleRouting-for-transaction-siz.patch (6.7K, ../../TYCPR01MB8373533A5C24BDDA516DA7E1ED9B9@TYCPR01MB8373.jpnprd01.prod.outlook.com/3-v13-0002-Export-PartitionTupleRouting-for-transaction-siz.patch) download | inline diff: From 84fb03e7cbb7c887ec926724c81c3baa5a7a7b9d Mon Sep 17 00:00:00 2001 From: Osumi Takamichi <[email protected]> Date: Thu, 18 Nov 2021 11:44:46 +0000 Subject: [PATCH v13 2/3] Export PartitionTupleRouting for transaction size calculation Apply worker utilizes ApplyExecutionData and its one of the members is PartitionTupleRouting. This is dynamically allocated during apply and need to make it exported if the target relation is partitioned to calculate transaction memory consumption. --- src/backend/executor/execPartition.c | 70 ------------------------------------ src/include/executor/execPartition.h | 69 +++++++++++++++++++++++++++++++++++ 2 files changed, 69 insertions(+), 70 deletions(-) diff --git a/src/backend/executor/execPartition.c b/src/backend/executor/execPartition.c index 5c723bc..2467f4c 100644 --- a/src/backend/executor/execPartition.c +++ b/src/backend/executor/execPartition.c @@ -34,76 +34,6 @@ #include "utils/rls.h" #include "utils/ruleutils.h" - -/*----------------------- - * PartitionTupleRouting - Encapsulates all information required to - * route a tuple inserted into a partitioned table to one of its leaf - * partitions. - * - * partition_root - * The partitioned table that's the target of the command. - * - * partition_dispatch_info - * Array of 'max_dispatch' elements containing a pointer to a - * PartitionDispatch object for every partitioned table touched by tuple - * routing. The entry for the target partitioned table is *always* - * present in the 0th element of this array. See comment for - * PartitionDispatchData->indexes for details on how this array is - * indexed. - * - * nonleaf_partitions - * Array of 'max_dispatch' elements containing pointers to fake - * ResultRelInfo objects for nonleaf partitions, useful for checking - * the partition constraint. - * - * num_dispatch - * The current number of items stored in the 'partition_dispatch_info' - * array. Also serves as the index of the next free array element for - * new PartitionDispatch objects that need to be stored. - * - * max_dispatch - * The current allocated size of the 'partition_dispatch_info' array. - * - * partitions - * Array of 'max_partitions' elements containing a pointer to a - * ResultRelInfo for every leaf partition touched by tuple routing. - * Some of these are pointers to ResultRelInfos which are borrowed out of - * the owning ModifyTableState node. The remainder have been built - * especially for tuple routing. See comment for - * PartitionDispatchData->indexes for details on how this array is - * indexed. - * - * is_borrowed_rel - * Array of 'max_partitions' booleans recording whether a given entry - * in 'partitions' is a ResultRelInfo pointer borrowed from the owning - * ModifyTableState node, rather than being built here. - * - * num_partitions - * The current number of items stored in the 'partitions' array. Also - * serves as the index of the next free array element for new - * ResultRelInfo objects that need to be stored. - * - * max_partitions - * The current allocated size of the 'partitions' array. - * - * memcxt - * Memory context used to allocate subsidiary structs. - *----------------------- - */ -struct PartitionTupleRouting -{ - Relation partition_root; - PartitionDispatch *partition_dispatch_info; - ResultRelInfo **nonleaf_partitions; - int num_dispatch; - int max_dispatch; - ResultRelInfo **partitions; - bool *is_borrowed_rel; - int num_partitions; - int max_partitions; - MemoryContext memcxt; -}; - /*----------------------- * PartitionDispatch - information about one partitioned table in a partition * hierarchy required to route a tuple to any of its partitions. A diff --git a/src/include/executor/execPartition.h b/src/include/executor/execPartition.h index 694e38b..4c796ab 100644 --- a/src/include/executor/execPartition.h +++ b/src/include/executor/execPartition.h @@ -110,6 +110,75 @@ typedef struct PartitionPruneState PartitionPruningData *partprunedata[FLEXIBLE_ARRAY_MEMBER]; } PartitionPruneState; +/*----------------------- + * PartitionTupleRouting - Encapsulates all information required to + * route a tuple inserted into a partitioned table to one of its leaf + * partitions. + * + * partition_root + * The partitioned table that's the target of the command. + * + * partition_dispatch_info + * Array of 'max_dispatch' elements containing a pointer to a + * PartitionDispatch object for every partitioned table touched by tuple + * routing. The entry for the target partitioned table is *always* + * present in the 0th element of this array. See comment for + * PartitionDispatchData->indexes for details on how this array is + * indexed. + * + * nonleaf_partitions + * Array of 'max_dispatch' elements containing pointers to fake + * ResultRelInfo objects for nonleaf partitions, useful for checking + * the partition constraint. + * + * num_dispatch + * The current number of items stored in the 'partition_dispatch_info' + * array. Also serves as the index of the next free array element for + * new PartitionDispatch objects that need to be stored. + * + * max_dispatch + * The current allocated size of the 'partition_dispatch_info' array. + * + * partitions + * Array of 'max_partitions' elements containing a pointer to a + * ResultRelInfo for every leaf partition touched by tuple routing. + * Some of these are pointers to ResultRelInfos which are borrowed out of + * the owning ModifyTableState node. The remainder have been built + * especially for tuple routing. See comment for + * PartitionDispatchData->indexes for details on how this array is + * indexed. + * + * is_borrowed_rel + * Array of 'max_partitions' booleans recording whether a given entry + * in 'partitions' is a ResultRelInfo pointer borrowed from the owning + * ModifyTableState node, rather than being built here. + * + * num_partitions + * The current number of items stored in the 'partitions' array. Also + * serves as the index of the next free array element for new + * ResultRelInfo objects that need to be stored. + * + * max_partitions + * The current allocated size of the 'partitions' array. + * + * memcxt + * Memory context used to allocate subsidiary structs. + *----------------------- + */ +struct PartitionTupleRouting +{ + Relation partition_root; + PartitionDispatch *partition_dispatch_info; + ResultRelInfo **nonleaf_partitions; + int num_dispatch; + int max_dispatch; + ResultRelInfo **partitions; + bool *is_borrowed_rel; + int num_partitions; + int max_partitions; + MemoryContext memcxt; +}; + extern PartitionTupleRouting *ExecSetupPartitionTupleRouting(EState *estate, Relation rel); extern ResultRelInfo *ExecFindPartition(ModifyTableState *mtstate, -- 2.2.0 [application/octet-stream] v13-0003-Extend-pg_stat_subscription_workers-to-include-g.patch (55.1K, ../../TYCPR01MB8373533A5C24BDDA516DA7E1ED9B9@TYCPR01MB8373.jpnprd01.prod.outlook.com/4-v13-0003-Extend-pg_stat_subscription_workers-to-include-g.patch) download | inline diff: From ac32a95ac67ca763a8e1cd14c7171804a88683dc Mon Sep 17 00:00:00 2001 From: Osumi Takamichi <[email protected]> Date: Thu, 18 Nov 2021 14:06:37 +0000 Subject: [PATCH v13 3/3] Extend pg_stat_subscription_workers to include general transaction statistics Categorize transactions of logical replication subscriber into three types (commit, error, abort) and introduce cumulative columns of those numbers and amounts of consumed data during message apply respectively. The calculation of consumed resources by subscriber's apply is computed based on the data structure for message apply, which is different from that of publisher's decoding processing which utilizes ReorderBufferChangeSize(). New bytes stats indicate how much resources apply worker used from the perspective of memory. Transaction size of table sync worker is not recorded as documented. At present, there's no special consideration to statistics of spool file logic such as amount of data spooled to disk, its count or corresponding statistics associated with STREAM ABORT. However, those can be added later. On subscriber, once either STREAM COMMIT message or STREAM PREPARE message arrives, apply worker starts to apply all spooled operations by apply_spooled_messages(). This replays transaction by reading entries. Therefore, size of such kind of message apply should be recorded into one of the columns introduced by this commit. The size of both PREPARE and STREAM PREPARE transaction become persistent to conclude the appropriate category for the transaction at either commit prepared or rollback prepared time, even after the server restart. Author: Takamichi Osumi Discussed & Reviewed-by: Amit Kapila, Masahiko Sawada, Hou Zhijie, Greg Nancarrow, Vignesh C, Ajin Cherian Discussion: https://www.postgresql.org/message-id/OSBPR01MB48887CA8F40C8D984A6DC00CED199%40OSBPR01MB4888.jpnprd01.prod.outlook.com --- doc/src/sgml/monitoring.sgml | 70 ++++- src/backend/catalog/system_views.sql | 6 + src/backend/postmaster/pgstat.c | 336 ++++++++++++++++++++++- src/backend/replication/logical/proto.c | 12 + src/backend/replication/logical/tablesync.c | 6 + src/backend/replication/logical/worker.c | 178 +++++++++++- src/backend/utils/adt/pgstatfuncs.c | 36 ++- src/include/catalog/pg_proc.dat | 6 +- src/include/pgstat.h | 101 ++++++- src/include/replication/logicalworker.h | 5 + src/test/regress/expected/rules.out | 8 +- src/test/subscription/t/027_worker_xact_stats.pl | 153 +++++++++++ src/tools/pgindent/typedefs.list | 4 + 13 files changed, 893 insertions(+), 28 deletions(-) create mode 100644 src/test/subscription/t/027_worker_xact_stats.pl diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml index 5b10d18..4944ef0 100644 --- a/doc/src/sgml/monitoring.sgml +++ b/doc/src/sgml/monitoring.sgml @@ -629,8 +629,8 @@ postgres 27093 0.0 0.0 30096 2752 ? Ss 11:34 0:00 postgres: ser <row> <entry><structname>pg_stat_subscription_workers</structname><indexterm><primary>pg_stat_subscription_workers</primary></indexterm></entry> - <entry>One row per subscription worker, showing statistics about errors - that occurred on that subscription worker. + <entry>One row per subscription worker, showing transaction statistics + and information about errors that occurred on that subscription worker. See <link linkend="monitoring-pg-stat-subscription-workers"> <structname>pg_stat_subscription_workers</structname></link> for details. </entry> @@ -3052,10 +3052,12 @@ SELECT pid, wait_event_type, wait_event FROM pg_stat_activity WHERE wait_event i <para> The <structname>pg_stat_subscription_workers</structname> view will contain - one row per subscription error reported by workers applying logical + one row per subscription, showing corresponding transaction statistics and + information about the error reported by workers applying logical replication changes and workers handling the initial data copy of the subscribed tables. The statistics entry is removed when the subscription - the worker is running on is removed. + the worker is running on is removed. The statistics of transaction size is + utilized only by the apply worker. </para> <table id="pg-stat-subscription-workers" xreflabel="pg_stat_subscription_workers"> @@ -3103,6 +3105,66 @@ SELECT pid, wait_event_type, wait_event FROM pg_stat_activity WHERE wait_event i <row> <entry role="catalog_table_entry"><para role="column_definition"> + <structfield>commit_count</structfield> <type>bigint</type> + </para> + <para> + Number of transactions successfully applied in this subscription. + COMMIT and COMMIT PREPARED increments this counter. + </para></entry> + </row> + + <row> + <entry role="catalog_table_entry"><para role="column_definition"> + <structfield>commit_bytes</structfield> <type>bigint</type> + </para> + <para> + Amount of data (in bytes) successfully applied in this subscription, + across <literal>commit_count</literal> transactions. + </para></entry> + </row> + + <row> + <entry role="catalog_table_entry"><para role="column_definition"> + <structfield>error_count</structfield> <type>bigint</type> + </para> + <para> + Number of transactions that failed to be applied by the table + sync worker or main apply worker in this subscription. + </para></entry> + </row> + + <row> + <entry role="catalog_table_entry"><para role="column_definition"> + <structfield>error_bytes</structfield> <type>bigint</type> + </para> + <para> + Amount of data (in bytes) unsuccessfully applied in this subscription, + across <literal>error_count</literal> transactions. + </para></entry> + </row> + + <row> + <entry role="catalog_table_entry"><para role="column_definition"> + <structfield>abort_count</structfield> <type>bigint</type> + </para> + <para> + Number of transactions aborted in this subscription. + ROLLBACK PREPARED increments this counter. + </para></entry> + </row> + + <row> + <entry role="catalog_table_entry"><para role="column_definition"> + <structfield>abort_bytes</structfield> <type>bigint</type> + </para> + <para> + Amount of data (in bytes) aborted in this subscription, + across <literal>abort_count</literal> transactions. + </para></entry> + </row> + + <row> + <entry role="catalog_table_entry"><para role="column_definition"> <structfield>last_error_relid</structfield> <type>oid</type> </para> <para> diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index ecf1a0b..a8880e8 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1267,6 +1267,12 @@ CREATE VIEW pg_stat_subscription_workers AS w.subid, s.subname, w.subrelid, + w.commit_count, + w.commit_bytes, + w.error_count, + w.error_bytes, + w.abort_count, + w.abort_bytes, w.last_error_relid, w.last_error_command, w.last_error_xid, diff --git a/src/backend/postmaster/pgstat.c b/src/backend/postmaster/pgstat.c index ee3b39a..017d9c8 100644 --- a/src/backend/postmaster/pgstat.c +++ b/src/backend/postmaster/pgstat.c @@ -53,6 +53,7 @@ #include "postmaster/fork_process.h" #include "postmaster/interrupt.h" #include "postmaster/postmaster.h" +#include "replication/logicalworker.h" #include "replication/slot.h" #include "replication/walsender.h" #include "storage/backendid.h" @@ -325,11 +326,14 @@ static PgStat_StatTabEntry *pgstat_get_tab_entry(PgStat_StatDBEntry *dbentry, static PgStat_StatSubWorkerEntry *pgstat_get_subworker_entry(PgStat_StatDBEntry *dbentry, Oid subid, Oid subrelid, bool create); +static PgStat_SW_PreparedXactEntry *pgstat_get_subworker_prepared_txn(Oid databaseid, Oid subid, + char *gid); + static void pgstat_write_statsfiles(bool permanent, bool allDbs); static void pgstat_write_db_statsfile(PgStat_StatDBEntry *dbentry, bool permanent); static HTAB *pgstat_read_statsfiles(Oid onlydb, bool permanent, bool deep); static void pgstat_read_db_statsfile(Oid databaseid, HTAB *tabhash, HTAB *funchash, - HTAB *subworkerhash, bool permanent); + HTAB *subworkerhash, HTAB *preparedtxnhash, bool permanent); static void backend_read_statsfile(void); static bool pgstat_write_statsfile_needed(void); @@ -382,6 +386,9 @@ static void pgstat_recv_replslot(PgStat_MsgReplSlot *msg, int len); static void pgstat_recv_tempfile(PgStat_MsgTempFile *msg, int len); static void pgstat_recv_subscription_purge(PgStat_MsgSubscriptionPurge *msg, int len); static void pgstat_recv_subworker_error(PgStat_MsgSubWorkerError *msg, int len); +static void pgstat_recv_subworker_xact_end(PgStat_MsgSubWorkerXactEnd *msg, int len); +static void pgstat_recv_subworker_twophase_xact(PgStat_MsgSubWorkerTwophaseXact *msg, int len); + /* ------------------------------------------------------------ * Public functions called from postmaster follow @@ -1930,6 +1937,61 @@ pgstat_report_replslot_drop(const char *slotname) } /* ---------- + * pgstat_report_subworker_xact_end() - + * + * Tell the collector that worker transaction has successfully completed. + * ---------- + */ +void +pgstat_report_subworker_xact_end(Oid subid, Oid subrel, + LogicalRepMsgType command, PgStat_Counter xact_size) +{ + PgStat_MsgSubWorkerXactEnd msg; + + Assert(command == 0 /* table sync worker */ || + command == LOGICAL_REP_MSG_COMMIT || + command == LOGICAL_REP_MSG_STREAM_COMMIT); + + pgstat_setheader(&msg.m_hdr, PGSTAT_MTYPE_SUBWORKERXACTEND); + msg.m_databaseid = MyDatabaseId; + msg.m_subid = subid; + msg.m_subrelid = subrel; + msg.m_command = command; + msg.m_xact_bytes = xact_size; + pgstat_send(&msg, sizeof(PgStat_MsgSubWorkerXactEnd)); + + reset_apply_xact_size(); +} + +/* ---------- + * pgstat_report_subworker_twophase_xact() - + * + * Tell the collector that worker transaction has done 2PC related operation. + * ---------- + */ +void +pgstat_report_subworker_twophase_xact(Oid subid, LogicalRepMsgType command, + PgStat_Counter xact_size, char *gid) +{ + PgStat_MsgSubWorkerTwophaseXact msg; + + Assert(command == LOGICAL_REP_MSG_PREPARE || + command == LOGICAL_REP_MSG_STREAM_PREPARE || + command == LOGICAL_REP_MSG_COMMIT_PREPARED || + command == LOGICAL_REP_MSG_ROLLBACK_PREPARED); + + /* setup the message */ + pgstat_setheader(&msg.m_hdr, PGSTAT_MTYPE_SUBWORKERTWOPHASEXACT); + msg.m_databaseid = MyDatabaseId; + msg.m_subid = subid; + msg.m_command = command; + strlcpy(msg.m_gid, gid, sizeof(msg.m_gid)); + msg.m_xact_bytes = xact_size; + pgstat_send(&msg, sizeof(PgStat_MsgSubWorkerTwophaseXact)); + reset_apply_xact_size(); +} + +/* ---------- * pgstat_report_subworker_error() - * * Tell the collector about the subscription worker error. @@ -1947,6 +2009,7 @@ pgstat_report_subworker_error(Oid subid, Oid subrelid, Oid relid, msg.m_databaseid = MyDatabaseId; msg.m_subid = subid; msg.m_subrelid = subrelid; + msg.m_xact_error_bytes = get_apply_xact_size(); msg.m_relid = relid; msg.m_command = command; msg.m_xid = xid; @@ -1955,6 +2018,8 @@ pgstat_report_subworker_error(Oid subid, Oid subrelid, Oid relid, len = offsetof(PgStat_MsgSubWorkerError, m_message) + strlen(msg.m_message) + 1; pgstat_send(&msg, len); + + reset_apply_xact_size(); } /* ---------- @@ -3725,6 +3790,14 @@ PgstatCollectorMain(int argc, char *argv[]) pgstat_recv_subworker_error(&msg.msg_subworkererror, len); break; + case PGSTAT_MTYPE_SUBWORKERXACTEND: + pgstat_recv_subworker_xact_end(&msg.msg_subworkerxactend, len); + break; + + case PGSTAT_MTYPE_SUBWORKERTWOPHASEXACT: + pgstat_recv_subworker_twophase_xact(&msg.msg_subworkertwophasexact, len); + break; + default: break; } @@ -3770,7 +3843,8 @@ PgstatCollectorMain(int argc, char *argv[]) /* * Subroutine to clear stats in a database entry * - * Tables, functions, and subscription workers hashes are initialized + * Tables, functions, subscription workers and subworkers + * prepared transaction size hashes are initialized * to empty. */ static void @@ -3831,6 +3905,14 @@ reset_dbentry_counters(PgStat_StatDBEntry *dbentry) PGSTAT_SUBWORKER_HASH_SIZE, &hash_ctl, HASH_ELEM | HASH_BLOBS); + + hash_ctl.keysize = sizeof(PgStat_SW_PreparedXactKey); + hash_ctl.entrysize = sizeof(PgStat_SW_PreparedXactEntry); + hash_ctl.hcxt = pgStatLocalContext; + dbentry->subworkers_preparedsizes = hash_create("Subscription worker stats of prepared txn", + PGSTAT_SUBWORKER_HASH_SIZE, + &hash_ctl, + HASH_ELEM | HASH_STRINGS | HASH_CONTEXT); } /* @@ -3855,7 +3937,8 @@ pgstat_get_db_entry(Oid databaseid, bool create) /* * If not found, initialize the new one. This creates empty hash tables - * for tables, functions, and subscription worker, too. + * for tables, functions, subscription worker and prepared transaction + * size of subscription worker, too. */ if (!found) reset_dbentry_counters(result); @@ -3941,6 +4024,12 @@ pgstat_get_subworker_entry(PgStat_StatDBEntry *dbentry, Oid subid, Oid subrelid, /* If not found, initialize the new one */ if (create && !found) { + subwentry->xact_commit_count = 0; + subwentry->xact_commit_bytes = 0; + subwentry->xact_error_count = 0; + subwentry->xact_error_bytes = 0; + subwentry->xact_abort_count = 0; + subwentry->xact_abort_bytes = 0; subwentry->relid = InvalidOid; subwentry->command = 0; subwentry->xid = InvalidTransactionId; @@ -4152,9 +4241,11 @@ pgstat_write_db_statsfile(PgStat_StatDBEntry *dbentry, bool permanent) HASH_SEQ_STATUS tstat; HASH_SEQ_STATUS fstat; HASH_SEQ_STATUS sstat; + HASH_SEQ_STATUS pstat; PgStat_StatTabEntry *tabentry; PgStat_StatFuncEntry *funcentry; PgStat_StatSubWorkerEntry *subwentry; + PgStat_SW_PreparedXactEntry *pentry; FILE *fpout; int32 format_id; Oid dbid = dbentry->databaseid; @@ -4221,6 +4312,17 @@ pgstat_write_db_statsfile(PgStat_StatDBEntry *dbentry, bool permanent) } /* + * Write subscription worker's prepared transaction struct + */ + hash_seq_init(&pstat, dbentry->subworkers_preparedsizes); + while ((pentry = (PgStat_SW_PreparedXactEntry *) hash_seq_search(&pstat)) != NULL) + { + fputc('P', fpout); + rc = fwrite(pentry, sizeof(PgStat_SW_PreparedXactEntry), 1, fpout); + (void) rc; /* we'll check for error with ferror */ + } + + /* * No more output to be done. Close the temp file and replace the old * pgstat.stat with it. The ferror() check replaces testing for error * after each individual fputc or fwrite above. @@ -4459,6 +4561,7 @@ pgstat_read_statsfiles(Oid onlydb, bool permanent, bool deep) dbentry->tables = NULL; dbentry->functions = NULL; dbentry->subworkers = NULL; + dbentry->subworkers_preparedsizes = NULL; /* * In the collector, disregard the timestamp we read from the @@ -4470,8 +4573,9 @@ pgstat_read_statsfiles(Oid onlydb, bool permanent, bool deep) dbentry->stats_timestamp = 0; /* - * Don't create tables/functions/subworkers hashtables for - * uninteresting databases. + * Don't create + * tables/functions/subworkers/subworkers_preparedsizes + * hashtables for uninteresting databases. */ if (onlydb != InvalidOid) { @@ -4504,6 +4608,14 @@ pgstat_read_statsfiles(Oid onlydb, bool permanent, bool deep) &hash_ctl, HASH_ELEM | HASH_BLOBS | HASH_CONTEXT); + hash_ctl.keysize = sizeof(PgStat_SW_PreparedXactKey); + hash_ctl.entrysize = sizeof(PgStat_SW_PreparedXactEntry); + hash_ctl.hcxt = pgStatLocalContext; + dbentry->subworkers_preparedsizes = hash_create("Subscription worker stats of prepared txn", + PGSTAT_SUBWORKER_HASH_SIZE, + &hash_ctl, + HASH_ELEM | HASH_STRINGS | HASH_CONTEXT); + /* * If requested, read the data from the database-specific * file. Otherwise we just leave the hashtables empty. @@ -4513,6 +4625,7 @@ pgstat_read_statsfiles(Oid onlydb, bool permanent, bool deep) dbentry->tables, dbentry->functions, dbentry->subworkers, + dbentry->subworkers_preparedsizes, permanent); break; @@ -4590,14 +4703,15 @@ done: * As in pgstat_read_statsfiles, if the permanent file is requested, it is * removed after reading. * - * Note: this code has the ability to skip storing per-table, per-function, or - * per-subscription-worker data, if NULL is passed for the corresponding hashtable. + * Note: this code has the ability to skip storing per-table, per-function, + * per-subscription-worker or per-subscription-prepared-size data, if NULL + * is passed for the corresponding hashtable. * That's not used at the moment though. * ---------- */ static void pgstat_read_db_statsfile(Oid databaseid, HTAB *tabhash, HTAB *funchash, - HTAB *subworkerhash, bool permanent) + HTAB *subworkerhash, HTAB *preparedtxnhash, bool permanent) { PgStat_StatTabEntry *tabentry; PgStat_StatTabEntry tabbuf; @@ -4753,6 +4867,32 @@ pgstat_read_db_statsfile(Oid databaseid, HTAB *tabhash, HTAB *funchash, memcpy(subwentry, &subwbuf, sizeof(subwbuf)); break; + case 'P': + { + PgStat_SW_PreparedXactEntry buff; + PgStat_SW_PreparedXactEntry *prepared_xact_entry; + + if (fread(&buff, 1, sizeof(PgStat_SW_PreparedXactEntry), + fpin) != sizeof(PgStat_SW_PreparedXactEntry)) + { + ereport(pgStatRunningInCollector ? LOG : WARNING, + (errmsg("corrupted statistics file \"%s\"", + statfile))); + goto done; + } + + if (preparedtxnhash == NULL) + break; + + prepared_xact_entry = + (PgStat_SW_PreparedXactEntry *) hash_search(preparedtxnhash, + (void *) &buff.key, + HASH_ENTER, NULL); + + memcpy(prepared_xact_entry, &buff, sizeof(PgStat_SW_PreparedXactEntry)); + break; + } + /* * 'E' The EOF marker of a complete stats file. */ @@ -5428,6 +5568,8 @@ pgstat_recv_dropdb(PgStat_MsgDropdb *msg, int len) hash_destroy(dbentry->functions); if (dbentry->subworkers != NULL) hash_destroy(dbentry->subworkers); + if (dbentry->subworkers_preparedsizes != NULL) + hash_destroy(dbentry->subworkers_preparedsizes); if (hash_search(pgStatDBHash, (void *) &dbid, @@ -5467,10 +5609,13 @@ pgstat_recv_resetcounter(PgStat_MsgResetcounter *msg, int len) hash_destroy(dbentry->functions); if (dbentry->subworkers != NULL) hash_destroy(dbentry->subworkers); + if (dbentry->subworkers_preparedsizes != NULL) + hash_destroy(dbentry->subworkers_preparedsizes); dbentry->tables = NULL; dbentry->functions = NULL; dbentry->subworkers = NULL; + dbentry->subworkers_preparedsizes = NULL; /* * Reset database-level stats, too. This creates empty hash tables for @@ -5546,10 +5691,28 @@ pgstat_recv_resetsinglecounter(PgStat_MsgResetsinglecounter *msg, int len) else if (msg->m_resettype == RESET_SUBWORKER) { PgStat_StatSubWorkerKey key; + PgStat_SW_PreparedXactKey pkey; key.subid = msg->m_objectid; key.subrelid = msg->m_subobjectid; (void) hash_search(dbentry->subworkers, (void *) &key, HASH_REMOVE, NULL); + + /* + * Clean up entry of prepared size hash as well. + * + * There's no gid input here as a part of key for + * PgStat_SW_PreparedXactEntry. Also any valid 'relid' indicates the + * entry is created by table sync but it has nothing to do with the + * two phase operation of apply. Proceed with this removal only when + * the specified subrelid isn't invalid. + */ + if (msg->m_subobjectid == InvalidOid) + { + pkey.subid = msg->m_objectid; + (void) hash_search(dbentry->subworkers_preparedsizes, + (void *) &pkey, HASH_REMOVE, NULL); + } + } } @@ -6105,6 +6268,7 @@ pgstat_recv_subscription_purge(PgStat_MsgSubscriptionPurge *msg, int len) HASH_SEQ_STATUS hstat; PgStat_StatDBEntry *dbentry; PgStat_StatSubWorkerEntry *subwentry; + PgStat_SW_PreparedXactEntry *prepared_txn_entry; dbentry = pgstat_get_db_entry(msg->m_databaseid, false); @@ -6126,6 +6290,127 @@ pgstat_recv_subscription_purge(PgStat_MsgSubscriptionPurge *msg, int len) } } } + + /* Remove associated prepared transaction stats */ + if (!dbentry->subworkers_preparedsizes) + { + hash_seq_init(&hstat, dbentry->subworkers_preparedsizes); + while ((prepared_txn_entry = (PgStat_SW_PreparedXactEntry *) hash_seq_search(&hstat)) != NULL) + { + for (int i = 0; i < msg->m_nentries; i++) + { + if (prepared_txn_entry->key.subid == msg->m_subids[i]) + (void) hash_search(dbentry->subworkers_preparedsizes, + (void *) &(prepared_txn_entry->key), + HASH_REMOVE, NULL); + break; + } + } + } +} + +/* ---------- + * pgstat_recv_subworker_xact_end() - + * + * Process a SUBWORKERXACTEND message. + * ---------- + */ +static void +pgstat_recv_subworker_xact_end(PgStat_MsgSubWorkerXactEnd *msg, int len) +{ + PgStat_StatDBEntry *dbentry; + PgStat_StatSubWorkerEntry *wentry; + + dbentry = pgstat_get_db_entry(msg->m_databaseid, true); + wentry = pgstat_get_subworker_entry(dbentry, msg->m_subid, + msg->m_subrelid, true); + Assert(wentry); + + /* table sync worker */ + if (msg->m_command == 0) + wentry->xact_commit_count++; + else + { + /* apply worker */ + switch (msg->m_command) + { + case LOGICAL_REP_MSG_COMMIT: + case LOGICAL_REP_MSG_STREAM_COMMIT: + wentry->xact_commit_count++; + wentry->xact_commit_bytes += msg->m_xact_bytes; + break; + default: + elog(ERROR, "unexpected logical message type as normal apply end"); + break; + } + } +} + +/* ---------- + * pgstat_recv_subworker_twophase_xact() - + * + * Process a SUBWORKERTWOPHASEXACT message. + * ---------- + */ +static void +pgstat_recv_subworker_twophase_xact(PgStat_MsgSubWorkerTwophaseXact *msg, int len) +{ + PgStat_StatDBEntry *dbentry; + PgStat_SW_PreparedXactEntry *pentry; + PgStat_StatSubWorkerEntry *wentry; + PgStat_SW_PreparedXactKey key; + + pentry = pgstat_get_subworker_prepared_txn(msg->m_databaseid, + msg->m_subid, + msg->m_gid); + Assert(pentry); + switch (msg->m_command) + { + case LOGICAL_REP_MSG_PREPARE: + case LOGICAL_REP_MSG_STREAM_PREPARE: + + /* + * Make each size of prepared transaction persistent so that we + * can update stats over the server restart and make prepared + * stats updated when commit prepared or rollback prepared + * arrives. + */ + pentry->subid = msg->m_subid; + strlcpy(pentry->gid, msg->m_gid, sizeof(pentry->gid)); + pentry->xact_size = msg->m_xact_bytes; + break; + + case LOGICAL_REP_MSG_COMMIT_PREPARED: + case LOGICAL_REP_MSG_ROLLBACK_PREPARED: + /* Update exported xact stats now */ + dbentry = pgstat_get_db_entry(msg->m_databaseid, true); + wentry = pgstat_get_subworker_entry(dbentry, + msg->m_subid, + InvalidOid /* apply worker */ , + true); + Assert(wentry); + if (msg->m_command == LOGICAL_REP_MSG_COMMIT_PREPARED) + { + wentry->xact_commit_count++; + wentry->xact_commit_bytes += pentry->xact_size; + } + else + { + wentry->xact_abort_count++; + wentry->xact_abort_bytes += pentry->xact_size; + } + + /* Clean up this gid from transaction size hash */ + key.subid = pentry->subid; + strlcpy(key.gid, msg->m_gid, sizeof(key.gid)); + (void) hash_search(dbentry->subworkers_preparedsizes, + (void *) &key, HASH_REMOVE, NULL); + break; + + default: + elog(ERROR, "unexpected logical message type as prepare transaction"); + break; + } } /* ---------- @@ -6162,6 +6447,8 @@ pgstat_recv_subworker_error(PgStat_MsgSubWorkerError *msg, int len) } /* Otherwise, update the error information */ + subwentry->xact_error_count++; + subwentry->xact_error_bytes += msg->m_xact_error_bytes; subwentry->relid = msg->m_relid; subwentry->command = msg->m_command; subwentry->xid = msg->m_xid; @@ -6394,3 +6681,36 @@ pgstat_count_slru_truncate(int slru_idx) { slru_entry(slru_idx)->m_truncate += 1; } + + /* ---------- + * pgstat_get_subworker_prepared_txn + * + * Return subscription worker entry with the given subscription OID and + * gid. + * ---------- + */ +static PgStat_SW_PreparedXactEntry * +pgstat_get_subworker_prepared_txn(Oid databaseid, Oid subid, + char *gid) +{ + PgStat_StatDBEntry *dbentry; + PgStat_SW_PreparedXactKey key; + PgStat_SW_PreparedXactEntry *pentry; + bool found; + + dbentry = pgstat_get_db_entry(databaseid, true); + key.subid = subid; + strlcpy(key.gid, gid, sizeof(key.gid)); + pentry = (PgStat_SW_PreparedXactEntry *) hash_search(dbentry->subworkers_preparedsizes, + (void *) &key, + HASH_ENTER, &found); + + if (!found) + { + pentry->subid = 0; + pentry->gid[0] = '\0'; + pentry->xact_size = 0; + } + + return pentry; +} diff --git a/src/backend/replication/logical/proto.c b/src/backend/replication/logical/proto.c index 9f5bf4b..8306876 100644 --- a/src/backend/replication/logical/proto.c +++ b/src/backend/replication/logical/proto.c @@ -17,6 +17,7 @@ #include "catalog/pg_type.h" #include "libpq/pqformat.h" #include "replication/logicalproto.h" +#include "replication/logicalworker.h" #include "utils/lsyscache.h" #include "utils/syscache.h" @@ -842,6 +843,7 @@ logicalrep_read_tuple(StringInfo in, LogicalRepTupleData *tuple) { int i; int natts; + Size total_size; /* Get number of attributes */ natts = pq_getmsgint(in, 2); @@ -850,6 +852,7 @@ logicalrep_read_tuple(StringInfo in, LogicalRepTupleData *tuple) tuple->colvalues = (StringInfoData *) palloc0(natts * sizeof(StringInfoData)); tuple->colstatus = (char *) palloc(natts * sizeof(char)); tuple->ncols = natts; + total_size = natts * (sizeof(char) + sizeof(StringInfoData)); /* Read the data */ for (i = 0; i < natts; i++) @@ -880,6 +883,9 @@ logicalrep_read_tuple(StringInfo in, LogicalRepTupleData *tuple) value->len = len; value->cursor = 0; value->maxlen = len; + + /* memory for tuple */ + total_size += len + 1; break; case LOGICALREP_COLUMN_BINARY: len = pq_getmsgint(in, 4); /* read length */ @@ -893,11 +899,17 @@ logicalrep_read_tuple(StringInfo in, LogicalRepTupleData *tuple) value->len = len; value->cursor = 0; value->maxlen = len; + + /* memory for tuple */ + total_size += len + 1; break; default: elog(ERROR, "unrecognized data representation type '%c'", kind); } } + + /* Record memory consumption for tuple */ + add_apply_xact_size(total_size); } /* diff --git a/src/backend/replication/logical/tablesync.c b/src/backend/replication/logical/tablesync.c index f07983a..5bb1d63 100644 --- a/src/backend/replication/logical/tablesync.c +++ b/src/backend/replication/logical/tablesync.c @@ -1149,6 +1149,12 @@ copy_table_done: MyLogicalRepWorker->relstate_lsn = *origin_startpos; SpinLockRelease(&MyLogicalRepWorker->relmutex); + /* Report the success of table sync. */ + pgstat_report_subworker_xact_end(MyLogicalRepWorker->subid, + MyLogicalRepWorker->relid, + 0 /* no logical message type */ , + 0 /* xact size */ ); + /* * Finally, wait until the main apply worker tells us to catch up and then * return to let LogicalRepApplyLoop do it. diff --git a/src/backend/replication/logical/worker.c b/src/backend/replication/logical/worker.c index 2e79302..c8d4255 100644 --- a/src/backend/replication/logical/worker.c +++ b/src/backend/replication/logical/worker.c @@ -215,12 +215,20 @@ typedef struct ApplyExecutionData PartitionTupleRouting *proute; /* partition routing info */ } ApplyExecutionData; -/* Struct for saving and restoring apply errcontext information */ +/* + * Struct for saving and restoring apply errcontext information + * + * In order not to lose transaction size in the case of error, + * have a member to store transaction memory consumption. + */ typedef struct ApplyErrorCallbackArg { LogicalRepMsgType command; /* 0 if invalid */ LogicalRepRelMapEntry *rel; + /* transaction size */ + PgStat_Counter bytes; + /* Remote node information */ int remote_attnum; /* -1 if invalid */ TransactionId remote_xid; @@ -231,6 +239,7 @@ static ApplyErrorCallbackArg apply_error_callback_arg = { .command = 0, .rel = NULL, + .bytes = 0, .remote_attnum = -1, .remote_xid = InvalidTransactionId, .ts = 0, @@ -304,6 +313,9 @@ static void maybe_reread_subscription(void); /* prototype needed because of stream_commit */ static void apply_dispatch(StringInfo s); +static void update_apply_change_size(LogicalRepMsgType action, + bool is_partitioned); + static void apply_handle_commit_internal(LogicalRepCommitData *commit_data); static void apply_handle_insert_internal(ApplyExecutionData *edata, ResultRelInfo *relinfo, @@ -818,6 +830,11 @@ apply_handle_commit(StringInfo s) /* Process any tables that are being synchronized in parallel. */ process_syncing_tables(commit_data.end_lsn); + pgstat_report_subworker_xact_end(MyLogicalRepWorker->subid, + InvalidOid, + LOGICAL_REP_MSG_COMMIT, + get_apply_xact_size()); + pgstat_report_activity(STATE_IDLE, NULL); reset_apply_error_context_info(); } @@ -922,6 +939,13 @@ apply_handle_prepare(StringInfo s) /* Process any tables that are being synchronized in parallel. */ process_syncing_tables(prepare_data.end_lsn); + /* Update the memory consumption of prepare and report */ + update_apply_change_size(LOGICAL_REP_MSG_PREPARE, false); + pgstat_report_subworker_twophase_xact(MyLogicalRepWorker->subid, + LOGICAL_REP_MSG_PREPARE, + get_apply_xact_size(), + prepare_data.gid); + pgstat_report_activity(STATE_IDLE, NULL); reset_apply_error_context_info(); } @@ -963,6 +987,13 @@ apply_handle_commit_prepared(StringInfo s) /* Process any tables that are being synchronized in parallel. */ process_syncing_tables(prepare_data.end_lsn); + /* Update the transaction size and report */ + update_apply_change_size(LOGICAL_REP_MSG_COMMIT_PREPARED, false); + pgstat_report_subworker_twophase_xact(MyLogicalRepWorker->subid, + LOGICAL_REP_MSG_COMMIT_PREPARED, + get_apply_xact_size(), + prepare_data.gid); + pgstat_report_activity(STATE_IDLE, NULL); reset_apply_error_context_info(); } @@ -1014,6 +1045,13 @@ apply_handle_rollback_prepared(StringInfo s) /* Process any tables that are being synchronized in parallel. */ process_syncing_tables(rollback_data.rollback_end_lsn); + /* Update the transaction size and report */ + update_apply_change_size(LOGICAL_REP_MSG_ROLLBACK_PREPARED, false); + pgstat_report_subworker_twophase_xact(MyLogicalRepWorker->subid, + LOGICAL_REP_MSG_ROLLBACK_PREPARED, + get_apply_xact_size(), + rollback_data.gid); + pgstat_report_activity(STATE_IDLE, NULL); reset_apply_error_context_info(); } @@ -1066,6 +1104,15 @@ apply_handle_stream_prepare(StringInfo s) /* Process any tables that are being synchronized in parallel. */ process_syncing_tables(prepare_data.end_lsn); + /* + * Report the prepared streaming xact size to the stats collector in a + * prepared xact manner to make it survive over the restart. + */ + pgstat_report_subworker_twophase_xact(MyLogicalRepWorker->subid, + LOGICAL_REP_MSG_STREAM_PREPARE, + get_apply_xact_size(), + prepare_data.gid); + pgstat_report_activity(STATE_IDLE, NULL); reset_apply_error_context_info(); @@ -1438,6 +1485,12 @@ apply_handle_stream_commit(StringInfo s) /* Process any tables that are being synchronized in parallel. */ process_syncing_tables(commit_data.end_lsn); + /* Report and clean up the xid */ + pgstat_report_subworker_xact_end(MyLogicalRepWorker->subid, + InvalidOid, + LOGICAL_REP_MSG_STREAM_COMMIT, + get_apply_xact_size()); + pgstat_report_activity(STATE_IDLE, NULL); reset_apply_error_context_info(); @@ -1579,6 +1632,10 @@ apply_handle_insert(StringInfo s) slot_fill_defaults(rel, estate, remoteslot); MemoryContextSwitchTo(oldctx); + /* Update transaction size */ + update_apply_change_size(LOGICAL_REP_MSG_INSERT, + rel->localrel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE); + /* For a partitioned table, insert the tuple into a partition. */ if (rel->localrel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE) apply_handle_tuple_routing(edata, @@ -1736,7 +1793,11 @@ apply_handle_update(StringInfo s) has_oldtup ? &oldtup : &newtup); MemoryContextSwitchTo(oldctx); - /* For a partitioned table, apply update to correct partition. */ + /* Update transaction size */ + update_apply_change_size(LOGICAL_REP_MSG_UPDATE, + rel->localrel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE); + + /* For a partitioned table, apply update to corect partition. */ if (rel->localrel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE) apply_handle_tuple_routing(edata, remoteslot, &newtup, CMD_UPDATE); @@ -1870,6 +1931,10 @@ apply_handle_delete(StringInfo s) slot_store_data(remoteslot, rel, &oldtup); MemoryContextSwitchTo(oldctx); + /* Update transaction size */ + update_apply_change_size(LOGICAL_REP_MSG_DELETE, + rel->localrel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE); + /* For a partitioned table, apply delete to correct partition. */ if (rel->localrel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE) apply_handle_tuple_routing(edata, @@ -2421,6 +2486,89 @@ apply_dispatch(StringInfo s) } /* + * Subscriber side implementation equivalent to ReorderBufferChangeSize + * of the publisher. + * + * The byte size of transaction on the publisher is calculated by + * ReorderBufferChangeSize() based on the ReorderBufferChange structure. + * But on the subscriber, consumed resources are not same as the + * publisher's decoding processsing and required to be computed in + * different way. Therefore, the exact same byte size is not restored on + * the subscriber usually. + * + * According to the logical replication message type, record major + * resource consumptions of this subscription for each message. + * At present, do not collect data from generic functions to keep + * code simplicity except for tuple length, since the implementation + * complexity versus benefit tradeoff should not be good. + * + * See logicalrep_read_tuple, in terms of tuple length consumption. + * + * Also, add multiple values at once in order to reduce the number + * of calls to this function, although the disadvantage of this way + * is we cannot get correct transaction size when we get an error. + * + * 'is_partitioned' is used to add some extra size. + */ +static void +update_apply_change_size(LogicalRepMsgType action, bool is_partitioned) +{ + int64 size = 0; + + switch (action) + { + /* No special memory consumption */ + case LOGICAL_REP_MSG_BEGIN: + case LOGICAL_REP_MSG_COMMIT: + case LOGICAL_REP_MSG_TRUNCATE: + case LOGICAL_REP_MSG_TYPE: + case LOGICAL_REP_MSG_ORIGIN: + case LOGICAL_REP_MSG_MESSAGE: + case LOGICAL_REP_MSG_BEGIN_PREPARE: + case LOGICAL_REP_MSG_RELATION: + break; + + case LOGICAL_REP_MSG_INSERT: + case LOGICAL_REP_MSG_UPDATE: + case LOGICAL_REP_MSG_DELETE: + Assert(!in_streamed_transaction); + + /* + * Compute size based on ApplyExecutionData. The size of + * LogicalRepRelMapEntry can be skipped because it is obtained + * from hash_search in logicalrep_rel_open. + */ + size += sizeof(ApplyExecutionData) + sizeof(EState) + + sizeof(ResultRelInfo) + sizeof(ResultRelInfo); + + /* Add some extra size if the target relation is partitioned. */ + if (is_partitioned) + size += sizeof(ModifyTableState) + sizeof(PartitionTupleRouting); + break; + + case LOGICAL_REP_MSG_PREPARE: + case LOGICAL_REP_MSG_COMMIT_PREPARED: + case LOGICAL_REP_MSG_ROLLBACK_PREPARED: + size += sizeof(FlushPosition); + break; + + case LOGICAL_REP_MSG_STREAM_START: + case LOGICAL_REP_MSG_STREAM_STOP: + case LOGICAL_REP_MSG_STREAM_COMMIT: + case LOGICAL_REP_MSG_STREAM_PREPARE: + case LOGICAL_REP_MSG_STREAM_ABORT: + default: + ereport(ERROR, + (errcode(ERRCODE_PROTOCOL_VIOLATION), + errmsg("invalid logical replication message type \"%c\"", action))); + } + + /* Update the total size of consumption when necessary */ + if (size != 0) + add_apply_xact_size(size); +} + +/* * Figure out which write/flush positions to report to the walsender process. * * We can't simply report back the last LSN the walsender sent us because the @@ -3605,6 +3753,32 @@ ApplyWorkerMain(Datum main_arg) } /* + * Get transaction size stored on apply error callback. + * This is used not only for error but also commit and + * rollback. Exported so that stats collector can utilize + * this value. + */ +int64 +get_apply_xact_size(void) +{ + return apply_error_callback_arg.bytes; +} + +/* Add size to apply transaction size */ +void +add_apply_xact_size(int64 size) +{ + apply_error_callback_arg.bytes += size; +} + +/* Reset transaction size on apply error callback */ +void +reset_apply_xact_size(void) +{ + apply_error_callback_arg.bytes = 0; +} + +/* * Is current process a logical replication worker? */ bool diff --git a/src/backend/utils/adt/pgstatfuncs.c b/src/backend/utils/adt/pgstatfuncs.c index a34c0b6..d7264fc 100644 --- a/src/backend/utils/adt/pgstatfuncs.c +++ b/src/backend/utils/adt/pgstatfuncs.c @@ -2414,7 +2414,7 @@ pg_stat_get_replication_slot(PG_FUNCTION_ARGS) Datum pg_stat_get_subscription_worker(PG_FUNCTION_ARGS) { -#define PG_STAT_GET_SUBSCRIPTION_WORKER_COLS 9 +#define PG_STAT_GET_SUBSCRIPTION_WORKER_COLS 15 Oid subid = PG_GETARG_OID(0); Oid subrelid; TupleDesc tupdesc; @@ -2441,19 +2441,31 @@ pg_stat_get_subscription_worker(PG_FUNCTION_ARGS) OIDOID, -1, 0); TupleDescInitEntry(tupdesc, (AttrNumber) 2, "subrelid", OIDOID, -1, 0); - TupleDescInitEntry(tupdesc, (AttrNumber) 3, "last_error_relid", + TupleDescInitEntry(tupdesc, (AttrNumber) 3, "commit_count", + INT8OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) 4, "commit_bytes", + INT8OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) 5, "error_count", + INT8OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) 6, "error_bytes", + INT8OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) 7, "abort_count", + INT8OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) 8, "abort_bytes", + INT8OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) 9, "last_error_relid", OIDOID, -1, 0); - TupleDescInitEntry(tupdesc, (AttrNumber) 4, "last_error_command", + TupleDescInitEntry(tupdesc, (AttrNumber) 10, "last_error_command", TEXTOID, -1, 0); - TupleDescInitEntry(tupdesc, (AttrNumber) 5, "last_error_xid", + TupleDescInitEntry(tupdesc, (AttrNumber) 11, "last_error_xid", XIDOID, -1, 0); - TupleDescInitEntry(tupdesc, (AttrNumber) 6, "last_error_count", + TupleDescInitEntry(tupdesc, (AttrNumber) 12, "last_error_count", INT8OID, -1, 0); - TupleDescInitEntry(tupdesc, (AttrNumber) 7, "last_error_message", + TupleDescInitEntry(tupdesc, (AttrNumber) 13, "last_error_message", TEXTOID, -1, 0); - TupleDescInitEntry(tupdesc, (AttrNumber) 8, "first_error_time", + TupleDescInitEntry(tupdesc, (AttrNumber) 14, "first_error_time", TIMESTAMPTZOID, -1, 0); - TupleDescInitEntry(tupdesc, (AttrNumber) 9, "last_error_time", + TupleDescInitEntry(tupdesc, (AttrNumber) 15, "last_error_time", TIMESTAMPTZOID, -1, 0); BlessTupleDesc(tupdesc); @@ -2471,6 +2483,14 @@ pg_stat_get_subscription_worker(PG_FUNCTION_ARGS) else nulls[i++] = true; + /* transaction stats */ + values[i++] = Int64GetDatum(wentry->xact_commit_count); + values[i++] = Int64GetDatum(wentry->xact_commit_bytes); + values[i++] = Int64GetDatum(wentry->xact_error_count); + values[i++] = Int64GetDatum(wentry->xact_error_bytes); + values[i++] = Int64GetDatum(wentry->xact_abort_count); + values[i++] = Int64GetDatum(wentry->xact_abort_bytes); + /* last_error_relid */ if (OidIsValid(wentry->relid)) values[i++] = ObjectIdGetDatum(wentry->relid); diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index 25f685f..fc654b4 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -5389,9 +5389,9 @@ proname => 'pg_stat_get_subscription_worker', prorows => '1', proisstrict => 'f', proretset => 't', provolatile => 's', proparallel => 'r', prorettype => 'record', proargtypes => 'oid oid', - proallargtypes => '{oid,oid,oid,oid,oid,text,xid,int8,text,timestamptz,timestamptz}', - proargmodes => '{i,i,o,o,o,o,o,o,o,o,o}', - proargnames => '{subid,subrelid,subid,subrelid,last_error_relid,last_error_command,last_error_xid,last_error_count,last_error_message,first_error_time,last_error_time}', + proallargtypes => '{oid,oid,oid,oid,int8,int8,int8,int8,int8,int8,oid,text,xid,int8,text,timestamptz,timestamptz}', + proargmodes => '{i,i,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o}', + proargnames => '{subid,subrelid,subid,subrelid,commit_count,commit_bytes,error_count,error_bytes,abort_count,abort_bytes,last_error_relid,last_error_command,last_error_xid,last_error_count,last_error_message,first_error_time,last_error_time}', prosrc => 'pg_stat_get_subscription_worker' }, { oid => '6118', descr => 'statistics: information about subscription', proname => 'pg_stat_get_subscription', prorows => '10', proisstrict => 'f', diff --git a/src/include/pgstat.h b/src/include/pgstat.h index 2c26b1c..266711b 100644 --- a/src/include/pgstat.h +++ b/src/include/pgstat.h @@ -86,6 +86,8 @@ typedef enum StatMsgType PGSTAT_MTYPE_DISCONNECT, PGSTAT_MTYPE_SUBSCRIPTIONPURGE, PGSTAT_MTYPE_SUBWORKERERROR, + PGSTAT_MTYPE_SUBWORKERXACTEND, + PGSTAT_MTYPE_SUBWORKERTWOPHASEXACT, } StatMsgType; /* ---------- @@ -558,6 +560,56 @@ typedef struct PgStat_MsgSubscriptionPurge } PgStat_MsgSubscriptionPurge; /* ---------- + * PgStat_MsgSubscriptionXactEnd Sent by the apply worker or the table sync worker + * to report successful transaction ends. + * ---------- + */ +typedef struct PgStat_MsgSubWorkerXactEnd +{ + PgStat_MsgHdr m_hdr; + + /* determine the worker entry */ + Oid m_databaseid; + Oid m_subid; + Oid m_subrelid; + + /* + * distinguish between transaction commits and streaming transaction + * aborts that are handled without error. + */ + LogicalRepMsgType m_command; + + /* memory consumption used by transaction */ + PgStat_Counter m_xact_bytes; + +} PgStat_MsgSubWorkerXactEnd; + +/* ---------- + * PgStat_MsgSubWorkerTwophaseXact Sent by the apply worker to make size of prepared + * txn persistent over the server restart and make it + * visible after commit prepare or rollback prepared. + * This is separated from PgStat_MsgSubWorkerXactEnd + * so that we can reduce message size of gid for other + * operations (e.g. normal COMMIT) that should happen more + * frequently than prepare operation usually. + * ---------- + */ +typedef struct PgStat_MsgSubWorkerTwophaseXact +{ + PgStat_MsgHdr m_hdr; + + /* determine the subscription */ + Oid m_databaseid; + Oid m_subid; + + LogicalRepMsgType m_command; + char m_gid[GIDSIZE]; + int gid_len; + PgStat_Counter m_xact_bytes; + +} PgStat_MsgSubWorkerTwophaseXact; + +/* ---------- * PgStat_MsgSubWorkerError Sent by the apply worker or the table sync worker to * report the error occurred during logical replication. * ---------- @@ -577,6 +629,12 @@ typedef struct PgStat_MsgSubWorkerError Oid m_subrelid; /* + * Transaction stats of subscription needs to be updated when an error + * occurs. + */ + PgStat_Counter m_xact_error_bytes; + + /* * Oid of the table that the reporter was actually processing. m_relid can * be InvalidOid if an error occurred during worker applying a * non-data-modification message such as RELATION. @@ -768,6 +826,8 @@ typedef union PgStat_Msg PgStat_MsgDisconnect msg_disconnect; PgStat_MsgSubscriptionPurge msg_subscriptionpurge; PgStat_MsgSubWorkerError msg_subworkererror; + PgStat_MsgSubWorkerXactEnd msg_subworkerxactend; + PgStat_MsgSubWorkerTwophaseXact msg_subworkertwophasexact; } PgStat_Msg; @@ -822,16 +882,23 @@ typedef struct PgStat_StatDBEntry TimestampTz stats_timestamp; /* time of db stats file update */ /* - * tables, functions, and subscription workers must be last in the struct, - * because we don't write the pointers out to the stats file. + * tables, functions, subscription workers and its prepared transaction + * stats must be last in the struct, because we don't write the pointers + * out to the stats file. * * subworker is the hash table of PgStat_StatSubWorkerEntry which stores * statistics of logical replication workers: apply worker and table sync * worker. + * + * subworkers_preparedsizes should give appropriate transaction sizes at + * either commit prepared or rollback prepared time, even when it's after + * the server restart. We have the apply worker send those statistics to + * the stats collector at prepare time. */ HTAB *tables; HTAB *functions; HTAB *subworkers; + HTAB *subworkers_preparedsizes; } PgStat_StatDBEntry; @@ -1005,6 +1072,16 @@ typedef struct PgStat_StatSubWorkerEntry PgStat_StatSubWorkerKey key; /* hash key (must be first) */ /* + * Cumulative transaction statistics of subscription worker + */ + PgStat_Counter xact_commit_count; + PgStat_Counter xact_commit_bytes; + PgStat_Counter xact_error_count; + PgStat_Counter xact_error_bytes; + PgStat_Counter xact_abort_count; + PgStat_Counter xact_abort_bytes; + + /* * Subscription worker error statistics representing an error that * occurred during application of logical replication or the initial table * synchronization. @@ -1018,6 +1095,22 @@ typedef struct PgStat_StatSubWorkerEntry char error_message[PGSTAT_SUBWORKERERROR_MSGLEN]; } PgStat_StatSubWorkerEntry; +/* prepared transaction */ +typedef struct PgStat_SW_PreparedXactKey +{ + Oid subid; + char gid[GIDSIZE]; +} PgStat_SW_PreparedXactKey; + +typedef struct PgStat_SW_PreparedXactEntry +{ + PgStat_SW_PreparedXactKey key; /* hash key */ + + Oid subid; + char gid[GIDSIZE]; + PgStat_Counter xact_size; +} PgStat_SW_PreparedXactEntry; + /* * Working state needed to accumulate per-function-call timing statistics. */ @@ -1128,6 +1221,10 @@ extern void pgstat_report_checksum_failure(void); extern void pgstat_report_replslot(const PgStat_StatReplSlotEntry *repSlotStat); extern void pgstat_report_replslot_create(const char *slotname); extern void pgstat_report_replslot_drop(const char *slotname); +extern void pgstat_report_subworker_xact_end(Oid subid, Oid subrel, + LogicalRepMsgType command, PgStat_Counter xact_size); +extern void pgstat_report_subworker_twophase_xact(Oid subid, LogicalRepMsgType command, + PgStat_Counter xact_size, char *gid); extern void pgstat_report_subworker_error(Oid subid, Oid subrelid, Oid relid, LogicalRepMsgType command, TransactionId xid, const char *errmsg); diff --git a/src/include/replication/logicalworker.h b/src/include/replication/logicalworker.h index 2ad61a0..435884d 100644 --- a/src/include/replication/logicalworker.h +++ b/src/include/replication/logicalworker.h @@ -16,4 +16,9 @@ extern void ApplyWorkerMain(Datum main_arg); extern bool IsLogicalWorker(void); +/* for transaction stats */ +extern int64 get_apply_xact_size(void); +extern void add_apply_xact_size(int64 size); +extern void reset_apply_xact_size(void); + #endif /* LOGICALWORKER_H */ diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out index d60c5a5..bd3e221 100644 --- a/src/test/regress/expected/rules.out +++ b/src/test/regress/expected/rules.out @@ -2097,6 +2097,12 @@ pg_stat_subscription| SELECT su.oid AS subid, pg_stat_subscription_workers| SELECT w.subid, s.subname, w.subrelid, + w.commit_count, + w.commit_bytes, + w.error_count, + w.error_bytes, + w.abort_count, + w.abort_bytes, w.last_error_relid, w.last_error_command, w.last_error_xid, @@ -2112,7 +2118,7 @@ pg_stat_subscription_workers| SELECT w.subid, pg_subscription_rel.srrelid AS relid FROM pg_subscription_rel WHERE (pg_subscription_rel.srsubstate <> 'r'::"char")) sr, - (LATERAL pg_stat_get_subscription_worker(sr.subid, sr.relid) w(subid, subrelid, last_error_relid, last_error_command, last_error_xid, last_error_count, last_error_message, first_error_time, last_error_time) + (LATERAL pg_stat_get_subscription_worker(sr.subid, sr.relid) w(subid, subrelid, commit_count, commit_bytes, error_count, error_bytes, abort_count, abort_bytes, last_error_relid, last_error_command, last_error_xid, last_error_count, last_error_message, first_error_time, last_error_time) JOIN pg_subscription s ON ((w.subid = s.oid))); pg_stat_sys_indexes| SELECT pg_stat_all_indexes.relid, pg_stat_all_indexes.indexrelid, diff --git a/src/test/subscription/t/027_worker_xact_stats.pl b/src/test/subscription/t/027_worker_xact_stats.pl new file mode 100644 index 0000000..3850408 --- /dev/null +++ b/src/test/subscription/t/027_worker_xact_stats.pl @@ -0,0 +1,153 @@ + +# Copyright (c) 2021, PostgreSQL Global Development Group + +# Tests for subscription worker statistics during apply. +use strict; +use warnings; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More tests => 2; + +# Create publisher node +my $node_publisher = PostgreSQL::Test::Cluster->new('publisher'); +$node_publisher->init(allows_streaming => 'logical'); +$node_publisher->append_conf( + 'postgresql.conf', qq[ +logical_decoding_work_mem = 64kB +max_prepared_transactions = 10 +max_wal_senders = 10 +wal_sender_timeout = 0 +]); +$node_publisher->start; + +# Create subscriber node +my $node_subscriber = PostgreSQL::Test::Cluster->new('subscriber'); +$node_subscriber->init(allows_streaming => 'logical'); +$node_subscriber->append_conf( + 'postgresql.conf', qq[ +max_prepared_transactions = 10 +]); +$node_subscriber->start; + +# Setup structures on the publisher and the subscriber +$node_publisher->safe_psql('postgres', "CREATE TABLE test_tab (a int)"); +$node_subscriber->safe_psql('postgres', + "CREATE TABLE test_tab (a int primary key)"); + +# Setup logical replication +my $publisher_connstr = $node_publisher->connstr . ' dbname=postgres'; +$node_publisher->safe_psql('postgres', + "CREATE PUBLICATION tap_pub FOR TABLE test_tab"); + +my $appname = 'tap_sub'; +my $subopts = 'streaming = on, two_phase = on, copy_data = false'; +$node_subscriber->safe_psql( + 'postgres', qq[ +CREATE SUBSCRIPTION tap_sub +CONNECTION '$publisher_connstr application_name=$appname' +PUBLICATION tap_pub WITH ($subopts); +]); + +$node_publisher->wait_for_catchup($appname); + +# There's no entry at the beginning +my $result = $node_subscriber->safe_psql('postgres', + "SELECT count(*) FROM pg_stat_subscription_workers"); +is($result, q(0), 'no entry for transaction stats yet'); + +# COMMIT +$node_publisher->safe_psql('postgres', + "BEGIN; INSERT INTO test_tab VALUES (1); COMMIT;"); + +$node_subscriber->poll_query_until('postgres', + "SELECT count(1) = 1 FROM pg_stat_subscription_workers where commit_count = 1" +) or die "didn't get updates of xact stats by commit"; + +# Now, stats collector make the bytes updated also. +$result = $node_subscriber->safe_psql('postgres', + "SELECT commit_bytes > 0 FROM pg_stat_subscription_workers"); +is($result, q(t), 'got consumed bytes'); + +# PREPARE & COMMIT PREPARED +$node_publisher->safe_psql('postgres', + "BEGIN; INSERT INTO test_tab VALUES (2); PREPARE TRANSACTION 'gid1'"); +$node_publisher->safe_psql('postgres', "COMMIT PREPARED 'gid1';"); +$node_subscriber->poll_query_until('postgres', + "SELECT count(1) = 1 FROM pg_stat_subscription_workers where commit_count = 2" +) or die "didn't get updates of xact stats by commit prepared"; + +# STREAM COMMIT +$node_publisher->safe_psql('postgres', + "BEGIN; INSERT INTO test_tab VALUES(generate_series(1001, 2000)); COMMIT;" +); +$node_subscriber->poll_query_until('postgres', + "SELECT count(1) = 1 FROM pg_stat_subscription_workers where commit_count = 3" +) or die "didn't get updates of xact stats by stream commit"; + +# STREAM PREPARE & COMMIT PREPARED +# This should restore the xact size before the shutdown. +$node_publisher->safe_psql( + 'postgres', q[ +BEGIN; +INSERT INTO test_tab VALUES(generate_series(2001, 3000)); +PREPARE TRANSACTION 'gid2'; +]); + +# This streamed prepare is not displayed until the commit prepared +# or rollback prepared. Hence, there's no way to confirm that +# stats collector has received the bytes of prepared transaction. +# So, instead of checking the view, issue one more committed transaction +# after the prepare and make sure that this commit's update is done, +# which should mean the previous streamed prepare is already processed +# by the stats collector as well. +$node_publisher->safe_psql('postgres', + "BEGIN; INSERT INTO test_tab VALUES (3); COMMIT;"); +$node_subscriber->poll_query_until('postgres', + "SELECT count(1) = 1 FROM pg_stat_subscription_workers where commit_count = 4" +) or die "didn't process the updates of committed transaction"; + +$node_subscriber->restart; + +# Get commit_bytes before commit prepared. +my $tmp = $node_subscriber->safe_psql('postgres', + "SELECT commit_bytes FROM pg_stat_subscription_workers where commit_count = 4" +); + +# Commit prepared increments the commit_count. +$node_publisher->safe_psql('postgres', "COMMIT PREPARED 'gid2'"); +$node_subscriber->poll_query_until('postgres', + "SELECT count(1) = 1 FROM pg_stat_subscription_workers where commit_count = 5" + ) + or die + "didn't get updates of xact stats by stream prepare and commit prepared"; + +$node_subscriber->poll_query_until('postgres', + "SELECT commit_bytes > $tmp FROM pg_stat_subscription_workers where commit_count = 5" +); + +# ROLLBACK PREPARED +$node_publisher->safe_psql( + 'postgres', q[ +BEGIN; +INSERT INTO test_tab VALUES (4); +PREPARE TRANSACTION 'gid3'; +]); +$node_publisher->safe_psql('postgres', "ROLLBACK PREPARED 'gid3'"); +$node_subscriber->poll_query_until('postgres', + "SELECT count(1) = 1 FROM pg_stat_subscription_workers where abort_count = 1" +) or die "didn't get updates of xact stats by rollback prepared"; + +# error stats (by duplication error) +$node_publisher->safe_psql( + 'postgres', q[ +BEGIN; +INSERT INTO test_tab VALUES (1); +COMMIT; +]); + +$node_subscriber->poll_query_until('postgres', + "SELECT count(1) = 1 FROM pg_stat_subscription_workers WHERE error_count > 0 AND error_bytes > 0" +) or die "didn't get updates of xact stats by error"; + +$node_subscriber->stop('fast'); +$node_publisher->stop('fast'); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f41ef0d..28154a3 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -1945,6 +1945,8 @@ PgStat_MsgResetslrucounter PgStat_MsgSLRU PgStat_MsgSubscriptionPurge PgStat_MsgSubWorkerError +PgStat_MsgSubWorkerTwophaseXact +PgStat_MsgSubWorkerXactEnd PgStat_MsgTabpurge PgStat_MsgTabstat PgStat_MsgTempFile @@ -1958,6 +1960,8 @@ PgStat_StatFuncEntry PgStat_StatReplSlotEntry PgStat_StatSubWorkerEntry PgStat_StatSubWorkerKey +PgStat_SW_PreparedXactEntry +PgStat_SW_PreparedXactKey PgStat_StatTabEntry PgStat_SubXactStatus PgStat_TableCounts -- 2.2.0 ^ permalink raw reply [nested|flat] 113+ messages in thread
* RE: Failed transaction statistics to measure the logical replication progress @ 2021-11-18 14:44 [email protected] <[email protected]> parent: Amit Kapila <[email protected]> 1 sibling, 0 replies; 113+ messages in thread From: [email protected] @ 2021-11-18 14:44 UTC (permalink / raw) To: 'Amit Kapila' <[email protected]>; +Cc: Masahiko Sawada <[email protected]>; vignesh C <[email protected]>; Greg Nancarrow <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]> On Thursday, November 18, 2021 12:26 PM Amit Kapila <[email protected]> wrote: > BTW, I think the way you are computing error_count in > pgstat_recv_subworker_error() doesn't seem correct to me because it will > accumulate the counter/bytes for the same error again and again. > You might want to update these counters after we have checked that the > received error is not the same as the previous one. Thank you for your comments ! This is addressed by v13 patchset [1] [1] - https://www.postgresql.org/message-id/TYCPR01MB8373533A5C24BDDA516DA7E1ED9B9%40TYCPR01MB8373.jpnprd0... Best Regards, Takamichi Osumi ^ permalink raw reply [nested|flat] 113+ messages in thread
* Re: Failed transaction statistics to measure the logical replication progress @ 2021-11-19 14:10 Masahiko Sawada <[email protected]> parent: Amit Kapila <[email protected]> 1 sibling, 4 replies; 113+ messages in thread From: Masahiko Sawada @ 2021-11-19 14:10 UTC (permalink / raw) To: Amit Kapila <[email protected]>; +Cc: [email protected] <[email protected]>; vignesh C <[email protected]>; Greg Nancarrow <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]> On Thu, Nov 18, 2021 at 12:26 PM Amit Kapila <[email protected]> wrote: > > On Wed, Nov 17, 2021 at 7:12 PM [email protected] > <[email protected]> wrote: > > > > On Wednesday, November 17, 2021 10:00 PM Amit Kapila <[email protected]> wrote: > > > > > > Can you please tell us why you think the names in your proposed patch are > > > better than the existing names proposed in Sawada-San's patch? Is it because > > > those fields always contain the information of the last or latest error that > > > occurred in the corresponding subscription worker? > > This is one reason. > > > > Another big reason comes from the final alignment when we list up all columns of both patches. > > The patches in this thread is trying to introduce a column that indicates > > cumulative count of error to show all error counts that the worker got in the past. > > > > Okay, I see your point and it makes sense to rename columns after > these other stats. I am not able to come up with any better names than > what is being used here. Sawada-San, do you agree with this, or do let > us know if you have any better ideas? > I'm concerned that these new names will introduce confusion; if we have last_error_relid, last_error_command, last_error_message, last_error_time, and last_error_xid, I think users might think that first_error_time is the timestamp at which an error occurred for the first time in the subscription worker. Also, I'm not sure last_error_count is not clear to me (it looks like showing something count but the only "last" one?). An alternative idea would be to add total_error_count by this patch, resulting in having both error_count and total_error_count. Regarding commit_count and abort_count, I personally think xact_commit and xact_rollback would be better since they’re more consistent with pg_stat_database view, although we might already have discussed that. Besides that, I’m not sure how useful commit_bytes, abort_bytes, and error_bytes are. I originally thought these statistics track the size of received data, i.g., how much data is transferred from the publisher and processed on the subscriber. But what the view currently has is how much memory is used in the subscription worker. The subscription worker emulates ReorderBufferChangeSize() on the subscriber side but, as the comment of update_apply_change_size() mentions, the size in the view is not accurate: + * The byte size of transaction on the publisher is calculated by + * ReorderBufferChangeSize() based on the ReorderBufferChange structure. + * But on the subscriber, consumed resources are not same as the + * publisher's decoding processsing and required to be computed in + * different way. Therefore, the exact same byte size is not restored on + * the subscriber usually. Also, it seems to take into account the size of FlushPosition that is not taken into account by ReorderBufferChangeSize(). I guess that the purpose of these values is to compare them to total_bytes, stream_byte, and spill_bytes but if the calculation is not accurate, does it mean that the more stats are updated, the more the stats will be getting inaccurate? Regards, -- Masahiko Sawada EDB: https://www.enterprisedb.com/ ^ permalink raw reply [nested|flat] 113+ messages in thread
* Re: Failed transaction statistics to measure the logical replication progress @ 2021-11-20 06:25 Amit Kapila <[email protected]> parent: Masahiko Sawada <[email protected]> 3 siblings, 1 reply; 113+ messages in thread From: Amit Kapila @ 2021-11-20 06:25 UTC (permalink / raw) To: Masahiko Sawada <[email protected]>; +Cc: [email protected] <[email protected]>; vignesh C <[email protected]>; Greg Nancarrow <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]> On Fri, Nov 19, 2021 at 7:41 PM Masahiko Sawada <[email protected]> wrote: > > On Thu, Nov 18, 2021 at 12:26 PM Amit Kapila <[email protected]> wrote: > > > > On Wed, Nov 17, 2021 at 7:12 PM [email protected] > > <[email protected]> wrote: > > > > > > On Wednesday, November 17, 2021 10:00 PM Amit Kapila <[email protected]> wrote: > > > > > > > > Can you please tell us why you think the names in your proposed patch are > > > > better than the existing names proposed in Sawada-San's patch? Is it because > > > > those fields always contain the information of the last or latest error that > > > > occurred in the corresponding subscription worker? > > > This is one reason. > > > > > > Another big reason comes from the final alignment when we list up all columns of both patches. > > > The patches in this thread is trying to introduce a column that indicates > > > cumulative count of error to show all error counts that the worker got in the past. > > > > > > > Okay, I see your point and it makes sense to rename columns after > > these other stats. I am not able to come up with any better names than > > what is being used here. Sawada-San, do you agree with this, or do let > > us know if you have any better ideas? > > > > I'm concerned that these new names will introduce confusion; if we > have last_error_relid, last_error_command, last_error_message, > last_error_time, and last_error_xid, I think users might think that > first_error_time is the timestamp at which an error occurred for the > first time in the subscription worker. > Isn't to some extent that confusion already exists because of last_error_time column? > Also, I'm not sure > last_error_count is not clear to me (it looks like showing something > count but the only "last" one?). > I feel if all the error related columns have "last_error_" as a prefix then it should not be that confusing? > An alternative idea would be to add > total_error_count by this patch, resulting in having both error_count > and total_error_count. Regarding commit_count and abort_count, I > personally think xact_commit and xact_rollback would be better since > they’re more consistent with pg_stat_database view, although we might > already have discussed that. > Even if we decide to change the column names to xact_commit/xact_rollback, I think with additional non-error columns it will be clear to add 'error' in column names corresponding to error columns, and last_error_* seems to be consistent with what we have in pg_stat_archiver (last_failed_wal, last_failed_time). Your point related to first_error_time has merit and I don't have a better answer for it. I think it is just a convenience column and we are not sure whether that will be required in practice so maybe we can drop that column and come back to it later once we get some field feedback on this view? -- With Regards, Amit Kapila. ^ permalink raw reply [nested|flat] 113+ messages in thread
* Re: Failed transaction statistics to measure the logical replication progress @ 2021-11-22 04:28 vignesh C <[email protected]> parent: Masahiko Sawada <[email protected]> 3 siblings, 0 replies; 113+ messages in thread From: vignesh C @ 2021-11-22 04:28 UTC (permalink / raw) To: Masahiko Sawada <[email protected]>; +Cc: Amit Kapila <[email protected]>; [email protected] <[email protected]>; Greg Nancarrow <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]> On Fri, Nov 19, 2021 at 7:41 PM Masahiko Sawada <[email protected]> wrote: > > On Thu, Nov 18, 2021 at 12:26 PM Amit Kapila <[email protected]> wrote: > > > > On Wed, Nov 17, 2021 at 7:12 PM [email protected] > > <[email protected]> wrote: > > > > > > On Wednesday, November 17, 2021 10:00 PM Amit Kapila <[email protected]> wrote: > > > > > > > > Can you please tell us why you think the names in your proposed patch are > > > > better than the existing names proposed in Sawada-San's patch? Is it because > > > > those fields always contain the information of the last or latest error that > > > > occurred in the corresponding subscription worker? > > > This is one reason. > > > > > > Another big reason comes from the final alignment when we list up all columns of both patches. > > > The patches in this thread is trying to introduce a column that indicates > > > cumulative count of error to show all error counts that the worker got in the past. > > > > > > > Okay, I see your point and it makes sense to rename columns after > > these other stats. I am not able to come up with any better names than > > what is being used here. Sawada-San, do you agree with this, or do let > > us know if you have any better ideas? > > > > I'm concerned that these new names will introduce confusion; if we > have last_error_relid, last_error_command, last_error_message, > last_error_time, and last_error_xid, I think users might think that > first_error_time is the timestamp at which an error occurred for the > first time in the subscription worker. Also, I'm not sure > last_error_count is not clear to me (it looks like showing something > count but the only "last" one?). An alternative idea would be to add > total_error_count by this patch, resulting in having both error_count > and total_error_count. Regarding commit_count and abort_count, I > personally think xact_commit and xact_rollback would be better since > they’re more consistent with pg_stat_database view, although we might > already have discussed that. > > Besides that, I’m not sure how useful commit_bytes, abort_bytes, and > error_bytes are. I originally thought these statistics track the size > of received data, i.g., how much data is transferred from the > publisher and processed on the subscriber. But what the view currently > has is how much memory is used in the subscription worker. The > subscription worker emulates ReorderBufferChangeSize() on the > subscriber side but, as the comment of update_apply_change_size() > mentions, the size in the view is not accurate: > > + * The byte size of transaction on the publisher is calculated by > + * ReorderBufferChangeSize() based on the ReorderBufferChange structure. > + * But on the subscriber, consumed resources are not same as the > + * publisher's decoding processsing and required to be computed in > + * different way. Therefore, the exact same byte size is not restored on > + * the subscriber usually. > > Also, it seems to take into account the size of FlushPosition that is > not taken into account by ReorderBufferChangeSize(). Let's keep the size calculation similar to the publisher side to avoid any confusion, we can try to keep it the same as ReorderBufferChangeSize wherever possible. Regards, Vignesh ^ permalink raw reply [nested|flat] 113+ messages in thread
* Re: Failed transaction statistics to measure the logical replication progress @ 2021-11-23 06:21 Greg Nancarrow <[email protected]> parent: Masahiko Sawada <[email protected]> 3 siblings, 1 reply; 113+ messages in thread From: Greg Nancarrow @ 2021-11-23 06:21 UTC (permalink / raw) To: Masahiko Sawada <[email protected]>; +Cc: Amit Kapila <[email protected]>; [email protected] <[email protected]>; vignesh C <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]> On Sat, Nov 20, 2021 at 1:11 AM Masahiko Sawada <[email protected]> wrote: > > I'm concerned that these new names will introduce confusion; if we > have last_error_relid, last_error_command, last_error_message, > last_error_time, and last_error_xid, I think users might think that > first_error_time is the timestamp at which an error occurred for the > first time in the subscription worker. You mean you think users might think "first_error_time" is the timestamp at which the last_error first occurred (rather than the timestamp of the first of any type of error that occurred) on that worker? > ... Also, I'm not sure > last_error_count is not clear to me (it looks like showing something > count but the only "last" one?). It's the number of times that the last_error has occurred. Unless it's some kind of transient error, that might get resolved without intervention, logical replication will get stuck in a loop retrying and the last error will occur again and again, hence the count of how many times that has happened. Maybe there's not much benefit in counting different errors prior to the last error? Regards, Greg Nancarrow Fujitsu Australia ^ permalink raw reply [nested|flat] 113+ messages in thread
* Re: Failed transaction statistics to measure the logical replication progress @ 2021-11-24 00:19 Masahiko Sawada <[email protected]> parent: Amit Kapila <[email protected]> 0 siblings, 0 replies; 113+ messages in thread From: Masahiko Sawada @ 2021-11-24 00:19 UTC (permalink / raw) To: Amit Kapila <[email protected]>; +Cc: [email protected] <[email protected]>; vignesh C <[email protected]>; Greg Nancarrow <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]> On Sat, Nov 20, 2021 at 3:25 PM Amit Kapila <[email protected]> wrote: > > On Fri, Nov 19, 2021 at 7:41 PM Masahiko Sawada <[email protected]> wrote: > > > > On Thu, Nov 18, 2021 at 12:26 PM Amit Kapila <[email protected]> wrote: > > > > > > On Wed, Nov 17, 2021 at 7:12 PM [email protected] > > > <[email protected]> wrote: > > > > > > > > On Wednesday, November 17, 2021 10:00 PM Amit Kapila <[email protected]> wrote: > > > > > > > > > > Can you please tell us why you think the names in your proposed patch are > > > > > better than the existing names proposed in Sawada-San's patch? Is it because > > > > > those fields always contain the information of the last or latest error that > > > > > occurred in the corresponding subscription worker? > > > > This is one reason. > > > > > > > > Another big reason comes from the final alignment when we list up all columns of both patches. > > > > The patches in this thread is trying to introduce a column that indicates > > > > cumulative count of error to show all error counts that the worker got in the past. > > > > > > > > > > Okay, I see your point and it makes sense to rename columns after > > > these other stats. I am not able to come up with any better names than > > > what is being used here. Sawada-San, do you agree with this, or do let > > > us know if you have any better ideas? > > > > > > > I'm concerned that these new names will introduce confusion; if we > > have last_error_relid, last_error_command, last_error_message, > > last_error_time, and last_error_xid, I think users might think that > > first_error_time is the timestamp at which an error occurred for the > > first time in the subscription worker. > > > > Isn't to some extent that confusion already exists because of > last_error_time column? > > > Also, I'm not sure > > last_error_count is not clear to me (it looks like showing something > > count but the only "last" one?). > > > > I feel if all the error related columns have "last_error_" as a prefix > then it should not be that confusing? > > > An alternative idea would be to add > > total_error_count by this patch, resulting in having both error_count > > and total_error_count. Regarding commit_count and abort_count, I > > personally think xact_commit and xact_rollback would be better since > > they’re more consistent with pg_stat_database view, although we might > > already have discussed that. > > > > Even if we decide to change the column names to > xact_commit/xact_rollback, I think with additional non-error columns > it will be clear to add 'error' in column names corresponding to error > columns, and last_error_* seems to be consistent with what we have in > pg_stat_archiver (last_failed_wal, last_failed_time). Okay, I agree that last_error_* columns will be consistent. > Your point > related to first_error_time has merit and I don't have a better answer > for it. I think it is just a convenience column and we are not sure > whether that will be required in practice so maybe we can drop that > column and come back to it later once we get some field feedback on > this view? +1 Regards, -- Masahiko Sawada EDB: https://www.enterprisedb.com/ ^ permalink raw reply [nested|flat] 113+ messages in thread
* Re: Failed transaction statistics to measure the logical replication progress @ 2021-11-24 00:26 Masahiko Sawada <[email protected]> parent: Greg Nancarrow <[email protected]> 0 siblings, 0 replies; 113+ messages in thread From: Masahiko Sawada @ 2021-11-24 00:26 UTC (permalink / raw) To: Greg Nancarrow <[email protected]>; +Cc: Amit Kapila <[email protected]>; [email protected] <[email protected]>; vignesh C <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]> On Tue, Nov 23, 2021 at 3:21 PM Greg Nancarrow <[email protected]> wrote: > > On Sat, Nov 20, 2021 at 1:11 AM Masahiko Sawada <[email protected]> wrote: > > > > I'm concerned that these new names will introduce confusion; if we > > have last_error_relid, last_error_command, last_error_message, > > last_error_time, and last_error_xid, I think users might think that > > first_error_time is the timestamp at which an error occurred for the > > first time in the subscription worker. > > You mean you think users might think "first_error_time" is the > timestamp at which the last_error first occurred (rather than the > timestamp of the first of any type of error that occurred) on that > worker? I felt that "first_error_time" is the timestamp of the first of any type of error that occurred on the worker. > > > ... Also, I'm not sure > > last_error_count is not clear to me (it looks like showing something > > count but the only "last" one?). > > It's the number of times that the last_error has occurred. > Unless it's some kind of transient error, that might get resolved > without intervention, logical replication will get stuck in a loop > retrying and the last error will occur again and again, hence the > count of how many times that has happened. > Maybe there's not much benefit in counting different errors prior to > the last error? The name "last_error_count" is somewhat clear to me now. I had felt that since the last error refers to *one* error that occurred last and it’s odd there is the count of it. Regards, -- Masahiko Sawada EDB: https://www.enterprisedb.com/ ^ permalink raw reply [nested|flat] 113+ messages in thread
* RE: Failed transaction statistics to measure the logical replication progress @ 2021-12-01 09:34 [email protected] <[email protected]> parent: Masahiko Sawada <[email protected]> 3 siblings, 1 reply; 113+ messages in thread From: [email protected] @ 2021-12-01 09:34 UTC (permalink / raw) To: 'Masahiko Sawada' <[email protected]>; Amit Kapila <[email protected]>; +Cc: vignesh C <[email protected]>; Greg Nancarrow <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]> On Friday, November 19, 2021 11:11 PM Masahiko Sawada <[email protected]> wrote: > Besides that, I’m not sure how useful commit_bytes, abort_bytes, and > error_bytes are. I originally thought these statistics track the size of received > data, i.g., how much data is transferred from the publisher and processed on > the subscriber. But what the view currently has is how much memory is used in > the subscription worker. The subscription worker emulates > ReorderBufferChangeSize() on the subscriber side but, as the comment of > update_apply_change_size() mentions, the size in the view is not accurate: ... > I guess that the purpose of these values is to compare them to total_bytes, > stream_byte, and spill_bytes but if the calculation is not accurate, does it mean > that the more stats are updated, the more the stats will be getting inaccurate? Thanks for your comment ! I tried to solve your concerns about byte columns but there are really difficult issues to solve. For example, to begin with the messages of apply worker are different from those of reorder buffer. Therefore, I decided to split the previous patch and make counter columns go first. v14 was checked by pgperltidy and pgindent. This patch can be applied to the PG whose commit id is after 8d74fc9 (introduction of pg_stat_subscription_workers). Best Regards, Takamichi Osumi Attachments: [application/octet-stream] v14-0001-Extend-pg_stat_subscription_workers-to-include-g.patch (22.5K, ../../TYCPR01MB8373380ED159E8BD52F2D9A5ED689@TYCPR01MB8373.jpnprd01.prod.outlook.com/2-v14-0001-Extend-pg_stat_subscription_workers-to-include-g.patch) download | inline diff: From 5869058de564558af1c84d185f74f24211d24035 Mon Sep 17 00:00:00 2001 From: Takamichi Osumi <[email protected]> Date: Wed, 1 Dec 2021 09:01:09 +0000 Subject: [PATCH v14] Extend pg_stat_subscription_workers to include general transaction statistics Categorize transactions of logical replication subscriber into three types (commit, error, abort) and introduce cumulative columns of those numbers. At present, there's no special consideration to statistics of spool file logic such as amount of data spooled to disk, its count or corresponding statistics associated with STREAM ABORT. However, those can be added later. Author: Takamichi Osumi Discussed & Reviewed-by: Amit Kapila, Masahiko Sawada, Hou Zhijie, Greg Nancarrow, Vignesh C, Ajin Cherian Discussion: https://www.postgresql.org/message-id/OSBPR01MB48887CA8F40C8D984A6DC00CED199%40OSBPR01MB4888.jpnprd01.prod.outlook.com --- doc/src/sgml/monitoring.sgml | 34 +++++- src/backend/catalog/system_views.sql | 3 + src/backend/postmaster/pgstat.c | 74 +++++++++++++ src/backend/replication/logical/tablesync.c | 5 + src/backend/replication/logical/worker.c | 16 +++ src/backend/utils/adt/pgstatfuncs.c | 25 +++-- src/include/catalog/pg_proc.dat | 6 +- src/include/pgstat.h | 29 +++++ src/test/regress/expected/rules.out | 5 +- src/test/subscription/t/027_worker_xact_stats.pl | 131 +++++++++++++++++++++++ src/tools/pgindent/typedefs.list | 1 + 11 files changed, 316 insertions(+), 13 deletions(-) create mode 100644 src/test/subscription/t/027_worker_xact_stats.pl diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml index 62f2a33..e10ca98 100644 --- a/doc/src/sgml/monitoring.sgml +++ b/doc/src/sgml/monitoring.sgml @@ -629,8 +629,8 @@ postgres 27093 0.0 0.0 30096 2752 ? Ss 11:34 0:00 postgres: ser <row> <entry><structname>pg_stat_subscription_workers</structname><indexterm><primary>pg_stat_subscription_workers</primary></indexterm></entry> - <entry>One row per subscription worker, showing statistics about errors - that occurred on that subscription worker. + <entry>One row per subscription worker, showing statistics about transactions + and errors that occurred on that subscription worker. See <link linkend="monitoring-pg-stat-subscription-workers"> <structname>pg_stat_subscription_workers</structname></link> for details. </entry> @@ -3123,6 +3123,36 @@ SELECT pid, wait_event_type, wait_event FROM pg_stat_activity WHERE wait_event i <row> <entry role="catalog_table_entry"><para role="column_definition"> + <structfield>commit_count</structfield> <type>bigint</type> + </para> + <para> + Number of transactions successfully applied in this subscription. + COMMIT and COMMIT PREPARED increments this counter. + </para></entry> + </row> + + <row> + <entry role="catalog_table_entry"><para role="column_definition"> + <structfield>error_count</structfield> <type>bigint</type> + </para> + <para> + Number of transactions that failed to be applied by the table + sync worker or main apply worker in this subscription. + </para></entry> + </row> + + <row> + <entry role="catalog_table_entry"><para role="column_definition"> + <structfield>abort_count</structfield> <type>bigint</type> + </para> + <para> + Number of transactions aborted in this subscription. + ROLLBACK PREPARED increments this counter. + </para></entry> + </row> + + <row> + <entry role="catalog_table_entry"><para role="column_definition"> <structfield>last_error_relid</structfield> <type>oid</type> </para> <para> diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index 61b515c..d2ca10a 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1267,6 +1267,9 @@ CREATE VIEW pg_stat_subscription_workers AS w.subid, s.subname, w.subrelid, + w.commit_count, + w.error_count, + w.abort_count, w.last_error_relid, w.last_error_command, w.last_error_xid, diff --git a/src/backend/postmaster/pgstat.c b/src/backend/postmaster/pgstat.c index 7264d2c..a6cb0e0 100644 --- a/src/backend/postmaster/pgstat.c +++ b/src/backend/postmaster/pgstat.c @@ -382,6 +382,7 @@ static void pgstat_recv_replslot(PgStat_MsgReplSlot *msg, int len); static void pgstat_recv_tempfile(PgStat_MsgTempFile *msg, int len); static void pgstat_recv_subscription_purge(PgStat_MsgSubscriptionPurge *msg, int len); static void pgstat_recv_subworker_error(PgStat_MsgSubWorkerError *msg, int len); +static void pgstat_recv_subworker_xact_end(PgStat_MsgSubWorkerXactEnd *msg, int len); /* ------------------------------------------------------------ * Public functions called from postmaster follow @@ -1949,6 +1950,31 @@ pgstat_report_replslot_drop(const char *slotname) } /* ---------- + * pgstat_report_subworker_xact_end() - + * + * Tell the collector that worker transaction has successfully completed. + * ---------- + */ +void +pgstat_report_subworker_xact_end(Oid subid, Oid subrel, LogicalRepMsgType command) +{ + PgStat_MsgSubWorkerXactEnd msg; + + Assert(command == 0 /* table sync worker */ || + command == LOGICAL_REP_MSG_COMMIT || + command == LOGICAL_REP_MSG_STREAM_COMMIT || + command == LOGICAL_REP_MSG_COMMIT_PREPARED || + command == LOGICAL_REP_MSG_ROLLBACK_PREPARED); + + pgstat_setheader(&msg.m_hdr, PGSTAT_MTYPE_SUBWORKERXACTEND); + msg.m_databaseid = MyDatabaseId; + msg.m_subid = subid; + msg.m_subrelid = subrel; + msg.m_command = command; + pgstat_send(&msg, sizeof(PgStat_MsgSubWorkerXactEnd)); +} + +/* ---------- * pgstat_report_subworker_error() - * * Tell the collector about the subscription worker error. @@ -3746,6 +3772,10 @@ PgstatCollectorMain(int argc, char *argv[]) pgstat_recv_subworker_error(&msg.msg_subworkererror, len); break; + case PGSTAT_MTYPE_SUBWORKERXACTEND: + pgstat_recv_subworker_xact_end(&msg.msg_subworkerxactend, len); + break; + default: break; } @@ -3965,6 +3995,9 @@ pgstat_get_subworker_entry(PgStat_StatDBEntry *dbentry, Oid subid, Oid subrelid, /* If not found, initialize the new one */ if (!found) { + subwentry->commit_count = 0; + subwentry->error_count = 0; + subwentry->abort_count = 0; subwentry->last_error_relid = InvalidOid; subwentry->last_error_command = 0; subwentry->last_error_xid = InvalidTransactionId; @@ -6153,6 +6186,46 @@ pgstat_recv_subscription_purge(PgStat_MsgSubscriptionPurge *msg, int len) } /* ---------- + * pgstat_recv_subworker_xact_end() - + * + * Process a SUBWORKERXACTEND message. + * ---------- + */ +static void +pgstat_recv_subworker_xact_end(PgStat_MsgSubWorkerXactEnd *msg, int len) +{ + PgStat_StatDBEntry *dbentry; + PgStat_StatSubWorkerEntry *wentry; + + dbentry = pgstat_get_db_entry(msg->m_databaseid, true); + wentry = pgstat_get_subworker_entry(dbentry, msg->m_subid, + msg->m_subrelid, true); + Assert(wentry); + + /* table sync worker */ + if (msg->m_command == 0) + wentry->commit_count++; + else + { + /* apply worker */ + switch (msg->m_command) + { + case LOGICAL_REP_MSG_COMMIT: + case LOGICAL_REP_MSG_STREAM_COMMIT: + case LOGICAL_REP_MSG_COMMIT_PREPARED: + wentry->commit_count++; + break; + case LOGICAL_REP_MSG_ROLLBACK_PREPARED: + wentry->abort_count++; + break; + default: + elog(ERROR, "unexpected logical message type as transaction end"); + break; + } + } +} + +/* ---------- * pgstat_recv_subworker_error() - * * Process a SUBWORKERERROR message. @@ -6186,6 +6259,7 @@ pgstat_recv_subworker_error(PgStat_MsgSubWorkerError *msg, int len) } /* Otherwise, update the error information */ + subwentry->error_count++; subwentry->last_error_relid = msg->m_relid; subwentry->last_error_command = msg->m_command; subwentry->last_error_xid = msg->m_xid; diff --git a/src/backend/replication/logical/tablesync.c b/src/backend/replication/logical/tablesync.c index f07983a..02e9486 100644 --- a/src/backend/replication/logical/tablesync.c +++ b/src/backend/replication/logical/tablesync.c @@ -1149,6 +1149,11 @@ copy_table_done: MyLogicalRepWorker->relstate_lsn = *origin_startpos; SpinLockRelease(&MyLogicalRepWorker->relmutex); + /* Report the success of table sync. */ + pgstat_report_subworker_xact_end(MyLogicalRepWorker->subid, + MyLogicalRepWorker->relid, + 0 /* no logical message type */ ); + /* * Finally, wait until the main apply worker tells us to catch up and then * return to let LogicalRepApplyLoop do it. diff --git a/src/backend/replication/logical/worker.c b/src/backend/replication/logical/worker.c index 2e79302..0a68de3 100644 --- a/src/backend/replication/logical/worker.c +++ b/src/backend/replication/logical/worker.c @@ -818,6 +818,10 @@ apply_handle_commit(StringInfo s) /* Process any tables that are being synchronized in parallel. */ process_syncing_tables(commit_data.end_lsn); + pgstat_report_subworker_xact_end(MyLogicalRepWorker->subid, + InvalidOid, + LOGICAL_REP_MSG_COMMIT); + pgstat_report_activity(STATE_IDLE, NULL); reset_apply_error_context_info(); } @@ -963,6 +967,10 @@ apply_handle_commit_prepared(StringInfo s) /* Process any tables that are being synchronized in parallel. */ process_syncing_tables(prepare_data.end_lsn); + pgstat_report_subworker_xact_end(MyLogicalRepWorker->subid, + InvalidOid, + LOGICAL_REP_MSG_COMMIT_PREPARED); + pgstat_report_activity(STATE_IDLE, NULL); reset_apply_error_context_info(); } @@ -1014,6 +1022,10 @@ apply_handle_rollback_prepared(StringInfo s) /* Process any tables that are being synchronized in parallel. */ process_syncing_tables(rollback_data.rollback_end_lsn); + pgstat_report_subworker_xact_end(MyLogicalRepWorker->subid, + InvalidOid, + LOGICAL_REP_MSG_ROLLBACK_PREPARED); + pgstat_report_activity(STATE_IDLE, NULL); reset_apply_error_context_info(); } @@ -1438,6 +1450,10 @@ apply_handle_stream_commit(StringInfo s) /* Process any tables that are being synchronized in parallel. */ process_syncing_tables(commit_data.end_lsn); + pgstat_report_subworker_xact_end(MyLogicalRepWorker->subid, + InvalidOid, + LOGICAL_REP_MSG_STREAM_COMMIT); + pgstat_report_activity(STATE_IDLE, NULL); reset_apply_error_context_info(); diff --git a/src/backend/utils/adt/pgstatfuncs.c b/src/backend/utils/adt/pgstatfuncs.c index f529c15..25a9fd2 100644 --- a/src/backend/utils/adt/pgstatfuncs.c +++ b/src/backend/utils/adt/pgstatfuncs.c @@ -2415,7 +2415,7 @@ pg_stat_get_replication_slot(PG_FUNCTION_ARGS) Datum pg_stat_get_subscription_worker(PG_FUNCTION_ARGS) { -#define PG_STAT_GET_SUBSCRIPTION_WORKER_COLS 8 +#define PG_STAT_GET_SUBSCRIPTION_WORKER_COLS 11 Oid subid = PG_GETARG_OID(0); Oid subrelid; TupleDesc tupdesc; @@ -2442,17 +2442,23 @@ pg_stat_get_subscription_worker(PG_FUNCTION_ARGS) OIDOID, -1, 0); TupleDescInitEntry(tupdesc, (AttrNumber) 2, "subrelid", OIDOID, -1, 0); - TupleDescInitEntry(tupdesc, (AttrNumber) 3, "last_error_relid", + TupleDescInitEntry(tupdesc, (AttrNumber) 3, "commit_count", + INT8OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) 4, "error_count", + INT8OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) 5, "abort_count", + INT8OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) 6, "last_error_relid", OIDOID, -1, 0); - TupleDescInitEntry(tupdesc, (AttrNumber) 4, "last_error_command", + TupleDescInitEntry(tupdesc, (AttrNumber) 7, "last_error_command", TEXTOID, -1, 0); - TupleDescInitEntry(tupdesc, (AttrNumber) 5, "last_error_xid", + TupleDescInitEntry(tupdesc, (AttrNumber) 8, "last_error_xid", XIDOID, -1, 0); - TupleDescInitEntry(tupdesc, (AttrNumber) 6, "last_error_count", + TupleDescInitEntry(tupdesc, (AttrNumber) 9, "last_error_count", INT8OID, -1, 0); - TupleDescInitEntry(tupdesc, (AttrNumber) 7, "last_error_message", + TupleDescInitEntry(tupdesc, (AttrNumber) 10, "last_error_message", TEXTOID, -1, 0); - TupleDescInitEntry(tupdesc, (AttrNumber) 8, "last_error_time", + TupleDescInitEntry(tupdesc, (AttrNumber) 11, "last_error_time", TIMESTAMPTZOID, -1, 0); BlessTupleDesc(tupdesc); @@ -2470,6 +2476,11 @@ pg_stat_get_subscription_worker(PG_FUNCTION_ARGS) else nulls[i++] = true; + /* transaction stats */ + values[i++] = Int64GetDatum(wentry->commit_count); + values[i++] = Int64GetDatum(wentry->error_count); + values[i++] = Int64GetDatum(wentry->abort_count); + /* last_error_relid */ if (OidIsValid(wentry->last_error_relid)) values[i++] = ObjectIdGetDatum(wentry->last_error_relid); diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index 79d787c..e78a4a2 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -5393,9 +5393,9 @@ proname => 'pg_stat_get_subscription_worker', prorows => '1', proisstrict => 'f', proretset => 't', provolatile => 's', proparallel => 'r', prorettype => 'record', proargtypes => 'oid oid', - proallargtypes => '{oid,oid,oid,oid,oid,text,xid,int8,text,timestamptz}', - proargmodes => '{i,i,o,o,o,o,o,o,o,o}', - proargnames => '{subid,subrelid,subid,subrelid,last_error_relid,last_error_command,last_error_xid,last_error_count,last_error_message,last_error_time}', + proallargtypes => '{oid,oid,oid,oid,int8,int8,int8,oid,text,xid,int8,text,timestamptz}', + proargmodes => '{i,i,o,o,o,o,o,o,o,o,o,o,o}', + proargnames => '{subid,subrelid,subid,subrelid,commit_count,error_count,abort_count,last_error_relid,last_error_command,last_error_xid,last_error_count,last_error_message,last_error_time}', prosrc => 'pg_stat_get_subscription_worker' }, { oid => '6118', descr => 'statistics: information about subscription', proname => 'pg_stat_get_subscription', prorows => '10', proisstrict => 'f', diff --git a/src/include/pgstat.h b/src/include/pgstat.h index 5b51b58..f976651 100644 --- a/src/include/pgstat.h +++ b/src/include/pgstat.h @@ -86,6 +86,7 @@ typedef enum StatMsgType PGSTAT_MTYPE_DISCONNECT, PGSTAT_MTYPE_SUBSCRIPTIONPURGE, PGSTAT_MTYPE_SUBWORKERERROR, + PGSTAT_MTYPE_SUBWORKERXACTEND, } StatMsgType; /* ---------- @@ -558,6 +559,25 @@ typedef struct PgStat_MsgSubscriptionPurge } PgStat_MsgSubscriptionPurge; /* ---------- + * PgStat_MsgSubscriptionXactEnd Sent by the apply worker or the table sync worker + * to report successful transaction ends. + * ---------- + */ +typedef struct PgStat_MsgSubWorkerXactEnd +{ + PgStat_MsgHdr m_hdr; + + /* determine the worker entry */ + Oid m_databaseid; + Oid m_subid; + Oid m_subrelid; + + /* necessary to determine column to increment */ + LogicalRepMsgType m_command; + +} PgStat_MsgSubWorkerXactEnd; + +/* ---------- * PgStat_MsgSubWorkerError Sent by the apply worker or the table sync * worker to report the error occurred while * processing changes. @@ -769,6 +789,7 @@ typedef union PgStat_Msg PgStat_MsgDisconnect msg_disconnect; PgStat_MsgSubscriptionPurge msg_subscriptionpurge; PgStat_MsgSubWorkerError msg_subworkererror; + PgStat_MsgSubWorkerXactEnd msg_subworkerxactend; } PgStat_Msg; @@ -1010,6 +1031,13 @@ typedef struct PgStat_StatSubWorkerEntry PgStat_StatSubWorkerKey key; /* hash key (must be first) */ /* + * Cumulative transaction statistics of subscription worker + */ + PgStat_Counter commit_count; + PgStat_Counter error_count; + PgStat_Counter abort_count; + + /* * Subscription worker error statistics representing an error that * occurred during application of changes or the initial table * synchronization. @@ -1131,6 +1159,7 @@ extern void pgstat_report_checksum_failure(void); extern void pgstat_report_replslot(const PgStat_StatReplSlotEntry *repSlotStat); extern void pgstat_report_replslot_create(const char *slotname); extern void pgstat_report_replslot_drop(const char *slotname); +extern void pgstat_report_subworker_xact_end(Oid subid, Oid subrel, LogicalRepMsgType command); extern void pgstat_report_subworker_error(Oid subid, Oid subrelid, Oid relid, LogicalRepMsgType command, TransactionId xid, const char *errmsg); diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out index b58b062..6f62f0d 100644 --- a/src/test/regress/expected/rules.out +++ b/src/test/regress/expected/rules.out @@ -2097,6 +2097,9 @@ pg_stat_subscription| SELECT su.oid AS subid, pg_stat_subscription_workers| SELECT w.subid, s.subname, w.subrelid, + w.commit_count, + w.error_count, + w.abort_count, w.last_error_relid, w.last_error_command, w.last_error_xid, @@ -2110,7 +2113,7 @@ pg_stat_subscription_workers| SELECT w.subid, SELECT pg_subscription_rel.srsubid AS subid, pg_subscription_rel.srrelid AS relid FROM pg_subscription_rel) sr, - (LATERAL pg_stat_get_subscription_worker(sr.subid, sr.relid) w(subid, subrelid, last_error_relid, last_error_command, last_error_xid, last_error_count, last_error_message, last_error_time) + (LATERAL pg_stat_get_subscription_worker(sr.subid, sr.relid) w(subid, subrelid, commit_count, error_count, abort_count, last_error_relid, last_error_command, last_error_xid, last_error_count, last_error_message, last_error_time) JOIN pg_subscription s ON ((w.subid = s.oid))); pg_stat_sys_indexes| SELECT pg_stat_all_indexes.relid, pg_stat_all_indexes.indexrelid, diff --git a/src/test/subscription/t/027_worker_xact_stats.pl b/src/test/subscription/t/027_worker_xact_stats.pl new file mode 100644 index 0000000..04e0abf --- /dev/null +++ b/src/test/subscription/t/027_worker_xact_stats.pl @@ -0,0 +1,131 @@ + +# Copyright (c) 2021, PostgreSQL Global Development Group + +# Tests for subscription worker statistics during apply. +use strict; +use warnings; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More tests => 1; + +# Create publisher node +my $node_publisher = PostgreSQL::Test::Cluster->new('publisher'); +$node_publisher->init(allows_streaming => 'logical'); +$node_publisher->append_conf( + 'postgresql.conf', qq[ +logical_decoding_work_mem = 64kB +max_prepared_transactions = 10 +max_wal_senders = 10 +wal_sender_timeout = 0 +]); +$node_publisher->start; + +# Create subscriber node +my $node_subscriber = PostgreSQL::Test::Cluster->new('subscriber'); +$node_subscriber->init(allows_streaming => 'logical'); +$node_subscriber->append_conf( + 'postgresql.conf', qq[ +max_prepared_transactions = 10 +]); +$node_subscriber->start; + +# Setup structures on the publisher and the subscriber +$node_publisher->safe_psql('postgres', "CREATE TABLE test_tab (a int)"); +$node_subscriber->safe_psql('postgres', + "CREATE TABLE test_tab (a int primary key)"); + +# Setup logical replication +my $publisher_connstr = $node_publisher->connstr . ' dbname=postgres'; +$node_publisher->safe_psql('postgres', + "CREATE PUBLICATION tap_pub FOR TABLE test_tab"); + +my $appname = 'tap_sub'; +my $subopts = 'streaming = on, two_phase = on, copy_data = false'; +$node_subscriber->safe_psql( + 'postgres', qq[ +CREATE SUBSCRIPTION tap_sub +CONNECTION '$publisher_connstr application_name=$appname' +PUBLICATION tap_pub WITH ($subopts); +]); + +$node_publisher->wait_for_catchup($appname); + +# There's no entry at the beginning +my $result = $node_subscriber->safe_psql('postgres', + "SELECT count(*) FROM pg_stat_subscription_workers"); +is($result, q(0), 'no entry for transaction stats yet'); + +# COMMIT +$node_publisher->safe_psql('postgres', + "BEGIN; INSERT INTO test_tab VALUES (1); COMMIT;"); + +$node_subscriber->poll_query_until('postgres', + "SELECT count(1) = 1 FROM pg_stat_subscription_workers where commit_count = 1" +) or die "didn't get updates of xact stats by commit"; + +# PREPARE & COMMIT PREPARED +$node_publisher->safe_psql('postgres', + "BEGIN; INSERT INTO test_tab VALUES (2); PREPARE TRANSACTION 'gid1'"); +$node_publisher->safe_psql('postgres', "COMMIT PREPARED 'gid1';"); +$node_subscriber->poll_query_until('postgres', + "SELECT count(1) = 1 FROM pg_stat_subscription_workers where commit_count = 2" +) or die "didn't get updates of xact stats by commit prepared"; + +# STREAM COMMIT +$node_publisher->safe_psql('postgres', + "BEGIN; INSERT INTO test_tab VALUES(generate_series(1001, 2000)); COMMIT;" +); +$node_subscriber->poll_query_until('postgres', + "SELECT count(1) = 1 FROM pg_stat_subscription_workers where commit_count = 3" +) or die "didn't get updates of xact stats by stream commit"; + +# STREAM PREPARE & COMMIT PREPARED +# This should restore the xact size before the shutdown. +$node_publisher->safe_psql( + 'postgres', q[ +BEGIN; +INSERT INTO test_tab VALUES(generate_series(2001, 3000)); +PREPARE TRANSACTION 'gid2'; +]); + +# Commit prepared increments the commit_count. +$node_publisher->safe_psql('postgres', "COMMIT PREPARED 'gid2'"); +$node_subscriber->poll_query_until('postgres', + "SELECT count(1) = 1 FROM pg_stat_subscription_workers where commit_count = 4" + ) + or die + "didn't get updates of xact stats by stream prepare and commit prepared"; + +# ROLLBACK PREPARED +$node_publisher->safe_psql( + 'postgres', q[ +BEGIN; +INSERT INTO test_tab VALUES (4); +PREPARE TRANSACTION 'gid3'; +]); +$node_publisher->safe_psql('postgres', "ROLLBACK PREPARED 'gid3'"); +$node_subscriber->poll_query_until('postgres', + "SELECT count(1) = 1 FROM pg_stat_subscription_workers where abort_count = 1" +) or die "didn't get updates of xact stats by rollback prepared"; + +# error stats (by duplication error) +$node_publisher->safe_psql( + 'postgres', q[ +BEGIN; +INSERT INTO test_tab VALUES (1); +COMMIT; +]); + +$node_subscriber->poll_query_until('postgres', + "SELECT count(1) = 1 FROM pg_stat_subscription_workers WHERE error_count > 0" +) or die "didn't get updates of xact stats by error"; + +# Check the cleanup of DROP SUBSCRIPTION +$node_subscriber->safe_psql('postgres', "DROP SUBSCRIPTION tap_sub"); + +$node_subscriber->poll_query_until('postgres', + "SELECT count(1) = 0 FROM pg_stat_subscription_workers") + or die "drop subscription failed to clean up the worker stats"; + +$node_subscriber->stop('fast'); +$node_publisher->stop('fast'); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f41ef0d..2d195ae 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -1945,6 +1945,7 @@ PgStat_MsgResetslrucounter PgStat_MsgSLRU PgStat_MsgSubscriptionPurge PgStat_MsgSubWorkerError +PgStat_MsgSubWorkerXactEnd PgStat_MsgTabpurge PgStat_MsgTabstat PgStat_MsgTempFile -- 1.8.3.1 ^ permalink raw reply [nested|flat] 113+ messages in thread
* Re: Failed transaction statistics to measure the logical replication progress @ 2021-12-03 06:11 vignesh C <[email protected]> parent: [email protected] <[email protected]> 0 siblings, 1 reply; 113+ messages in thread From: vignesh C @ 2021-12-03 06:11 UTC (permalink / raw) To: [email protected] <[email protected]>; +Cc: Masahiko Sawada <[email protected]>; Amit Kapila <[email protected]>; Greg Nancarrow <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]> On Wed, Dec 1, 2021 at 3:04 PM [email protected] <[email protected]> wrote: > > On Friday, November 19, 2021 11:11 PM Masahiko Sawada <[email protected]> wrote: > > Besides that, I’m not sure how useful commit_bytes, abort_bytes, and > > error_bytes are. I originally thought these statistics track the size of received > > data, i.g., how much data is transferred from the publisher and processed on > > the subscriber. But what the view currently has is how much memory is used in > > the subscription worker. The subscription worker emulates > > ReorderBufferChangeSize() on the subscriber side but, as the comment of > > update_apply_change_size() mentions, the size in the view is not accurate: > ... > > I guess that the purpose of these values is to compare them to total_bytes, > > stream_byte, and spill_bytes but if the calculation is not accurate, does it mean > > that the more stats are updated, the more the stats will be getting inaccurate? > Thanks for your comment ! > > I tried to solve your concerns about byte columns but there are really difficult issues to solve. > For example, to begin with the messages of apply worker are different from those of > reorder buffer. > > Therefore, I decided to split the previous patch and make counter columns go first. > v14 was checked by pgperltidy and pgindent. > > This patch can be applied to the PG whose commit id is after 8d74fc9 (introduction of > pg_stat_subscription_workers). Thanks for the updated patch. Currently we are storing the commit count, error_count and abort_count for each table of the table sync operation. If we have thousands of tables, we will be storing the information for each of the tables. Shouldn't we be storing the consolidated information in this case. diff --git a/src/backend/replication/logical/tablesync.c b/src/backend/replication/logical/tablesync.c index f07983a..02e9486 100644 --- a/src/backend/replication/logical/tablesync.c +++ b/src/backend/replication/logical/tablesync.c @@ -1149,6 +1149,11 @@ copy_table_done: MyLogicalRepWorker->relstate_lsn = *origin_startpos; SpinLockRelease(&MyLogicalRepWorker->relmutex); + /* Report the success of table sync. */ + pgstat_report_subworker_xact_end(MyLogicalRepWorker->subid, + MyLogicalRepWorker->relid, + 0 /* no logical message type */ ); postgres=# select * from pg_stat_subscription_workers ; subid | subname | subrelid | commit_count | error_count | abort_count | last_error_relid | last_error_command | last_error_xid | last_error_count | last_error_message | last_error_time -------+---------+----------+--------------+-------------+-------------+------------------+--------------------+----------------+------------------+--------------------+----------------- 16411 | sub1 | 16387 | 1 | 0 | 0 | | | | 0 | | 16411 | sub1 | 16396 | 1 | 0 | 0 | | | | 0 | | 16411 | sub1 | 16390 | 1 | 0 | 0 | | | | 0 | | 16411 | sub1 | 16393 | 1 | 0 | 0 | | | | 0 | | 16411 | sub1 | 16402 | 1 | 0 | 0 | | | | 0 | | 16411 | sub1 | 16408 | 1 | 0 | 0 | | | | 0 | | 16411 | sub1 | 16384 | 1 | 0 | 0 | | | | 0 | | 16411 | sub1 | 16399 | 1 | 0 | 0 | | | | 0 | | 16411 | sub1 | 16405 | 1 | 0 | 0 | | | | 0 | | (9 rows) Regards, Vignesh ^ permalink raw reply [nested|flat] 113+ messages in thread
* RE: Failed transaction statistics to measure the logical replication progress @ 2021-12-04 13:01 [email protected] <[email protected]> parent: vignesh C <[email protected]> 0 siblings, 1 reply; 113+ messages in thread From: [email protected] @ 2021-12-04 13:01 UTC (permalink / raw) To: 'vignesh C' <[email protected]>; +Cc: Masahiko Sawada <[email protected]>; Amit Kapila <[email protected]>; Greg Nancarrow <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]> On Friday, December 3, 2021 3:12 PM vignesh C <[email protected]> wrote: > Thanks for the updated patch. > Currently we are storing the commit count, error_count and abort_count for > each table of the table sync operation. If we have thousands of tables, we will > be storing the information for each of the tables. > Shouldn't we be storing the consolidated information in this case. > diff --git a/src/backend/replication/logical/tablesync.c > b/src/backend/replication/logical/tablesync.c > index f07983a..02e9486 100644 > --- a/src/backend/replication/logical/tablesync.c > +++ b/src/backend/replication/logical/tablesync.c > @@ -1149,6 +1149,11 @@ copy_table_done: > MyLogicalRepWorker->relstate_lsn = *origin_startpos; > SpinLockRelease(&MyLogicalRepWorker->relmutex); > > + /* Report the success of table sync. */ > + pgstat_report_subworker_xact_end(MyLogicalRepWorker->subid, > + > MyLogicalRepWorker->relid, > + > 0 /* no logical message type */ ); Okay. I united all stats into that of apply worker. In line with this change, I fixed the TAP tests as well to cover the updates of stats done by table sync workers. Also, during my self-review, I noticed that I should call pgstat_report_subworker_xact_end() before process_syncing_tables() because it can lead to process exit, which results in missing one increment of the stats columns. I noted this point in a comment as well. Best Regards, Takamichi Osumi Attachments: [application/octet-stream] v15-0001-Extend-pg_stat_subscription_workers-to-include-g.patch (24.3K, ../../TYWPR01MB8362480F9148248F3D1B1213ED6B9@TYWPR01MB8362.jpnprd01.prod.outlook.com/2-v15-0001-Extend-pg_stat_subscription_workers-to-include-g.patch) download | inline diff: From 1b756a897eee84e068e039333666278112c235b0 Mon Sep 17 00:00:00 2001 From: Takamichi Osumi <[email protected]> Date: Sat, 4 Dec 2021 12:27:10 +0000 Subject: [PATCH v15] Extend pg_stat_subscription_workers to include general transaction statistics Categorize transactions of logical replication subscriber into three types (commit, error, abort) and introduce cumulative columns of those numbers. The stats for table sync worker is merged into those of the apply worker. At present, there's no special consideration to statistics of spool file logic such as amount of data spooled to disk, its count or corresponding statistics associated with STREAM ABORT. However, those can be added later. Author: Takamichi Osumi Reviewed-by: Amit Kapila, Masahiko Sawada, Hou Zhijie, Greg Nancarrow, Vignesh C, Ajin Cherian Discussion: https://www.postgresql.org/message-id/OSBPR01MB48887CA8F40C8D984A6DC00CED199%40OSBPR01MB4888.jpnprd01.prod.outlook.com --- doc/src/sgml/monitoring.sgml | 34 ++++- src/backend/catalog/system_views.sql | 3 + src/backend/postmaster/pgstat.c | 79 +++++++++++ src/backend/replication/logical/tablesync.c | 7 + src/backend/replication/logical/worker.c | 20 +++ src/backend/utils/adt/pgstatfuncs.c | 25 +++- src/include/catalog/pg_proc.dat | 6 +- src/include/pgstat.h | 29 ++++ src/test/regress/expected/rules.out | 5 +- src/test/subscription/t/027_worker_xact_stats.pl | 162 +++++++++++++++++++++++ src/tools/pgindent/typedefs.list | 1 + 11 files changed, 358 insertions(+), 13 deletions(-) create mode 100644 src/test/subscription/t/027_worker_xact_stats.pl diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml index 62f2a33..e10ca98 100644 --- a/doc/src/sgml/monitoring.sgml +++ b/doc/src/sgml/monitoring.sgml @@ -629,8 +629,8 @@ postgres 27093 0.0 0.0 30096 2752 ? Ss 11:34 0:00 postgres: ser <row> <entry><structname>pg_stat_subscription_workers</structname><indexterm><primary>pg_stat_subscription_workers</primary></indexterm></entry> - <entry>One row per subscription worker, showing statistics about errors - that occurred on that subscription worker. + <entry>One row per subscription worker, showing statistics about transactions + and errors that occurred on that subscription worker. See <link linkend="monitoring-pg-stat-subscription-workers"> <structname>pg_stat_subscription_workers</structname></link> for details. </entry> @@ -3123,6 +3123,36 @@ SELECT pid, wait_event_type, wait_event FROM pg_stat_activity WHERE wait_event i <row> <entry role="catalog_table_entry"><para role="column_definition"> + <structfield>commit_count</structfield> <type>bigint</type> + </para> + <para> + Number of transactions successfully applied in this subscription. + COMMIT and COMMIT PREPARED increments this counter. + </para></entry> + </row> + + <row> + <entry role="catalog_table_entry"><para role="column_definition"> + <structfield>error_count</structfield> <type>bigint</type> + </para> + <para> + Number of transactions that failed to be applied by the table + sync worker or main apply worker in this subscription. + </para></entry> + </row> + + <row> + <entry role="catalog_table_entry"><para role="column_definition"> + <structfield>abort_count</structfield> <type>bigint</type> + </para> + <para> + Number of transactions aborted in this subscription. + ROLLBACK PREPARED increments this counter. + </para></entry> + </row> + + <row> + <entry role="catalog_table_entry"><para role="column_definition"> <structfield>last_error_relid</structfield> <type>oid</type> </para> <para> diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index 61b515c..d2ca10a 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1267,6 +1267,9 @@ CREATE VIEW pg_stat_subscription_workers AS w.subid, s.subname, w.subrelid, + w.commit_count, + w.error_count, + w.abort_count, w.last_error_relid, w.last_error_command, w.last_error_xid, diff --git a/src/backend/postmaster/pgstat.c b/src/backend/postmaster/pgstat.c index 7264d2c..5fe49ae 100644 --- a/src/backend/postmaster/pgstat.c +++ b/src/backend/postmaster/pgstat.c @@ -382,6 +382,7 @@ static void pgstat_recv_replslot(PgStat_MsgReplSlot *msg, int len); static void pgstat_recv_tempfile(PgStat_MsgTempFile *msg, int len); static void pgstat_recv_subscription_purge(PgStat_MsgSubscriptionPurge *msg, int len); static void pgstat_recv_subworker_error(PgStat_MsgSubWorkerError *msg, int len); +static void pgstat_recv_subworker_xact_end(PgStat_MsgSubWorkerXactEnd *msg, int len); /* ------------------------------------------------------------ * Public functions called from postmaster follow @@ -1949,6 +1950,30 @@ pgstat_report_replslot_drop(const char *slotname) } /* ---------- + * pgstat_report_subworker_xact_end() - + * + * Tell the collector that worker transaction has successfully completed. + * ---------- + */ +void +pgstat_report_subworker_xact_end(Oid subid, LogicalRepMsgType command) +{ + PgStat_MsgSubWorkerXactEnd msg; + + /* LOGICAL_REP_MSG_COMMIT cares about table sync's commit too */ + Assert(command == LOGICAL_REP_MSG_COMMIT || + command == LOGICAL_REP_MSG_STREAM_COMMIT || + command == LOGICAL_REP_MSG_COMMIT_PREPARED || + command == LOGICAL_REP_MSG_ROLLBACK_PREPARED); + + pgstat_setheader(&msg.m_hdr, PGSTAT_MTYPE_SUBWORKERXACTEND); + msg.m_databaseid = MyDatabaseId; + msg.m_subid = subid; + msg.m_command = command; + pgstat_send(&msg, sizeof(PgStat_MsgSubWorkerXactEnd)); +} + +/* ---------- * pgstat_report_subworker_error() - * * Tell the collector about the subscription worker error. @@ -3746,6 +3771,10 @@ PgstatCollectorMain(int argc, char *argv[]) pgstat_recv_subworker_error(&msg.msg_subworkererror, len); break; + case PGSTAT_MTYPE_SUBWORKERXACTEND: + pgstat_recv_subworker_xact_end(&msg.msg_subworkerxactend, len); + break; + default: break; } @@ -3965,6 +3994,9 @@ pgstat_get_subworker_entry(PgStat_StatDBEntry *dbentry, Oid subid, Oid subrelid, /* If not found, initialize the new one */ if (!found) { + subwentry->commit_count = 0; + subwentry->error_count = 0; + subwentry->abort_count = 0; subwentry->last_error_relid = InvalidOid; subwentry->last_error_command = 0; subwentry->last_error_xid = InvalidTransactionId; @@ -6153,6 +6185,39 @@ pgstat_recv_subscription_purge(PgStat_MsgSubscriptionPurge *msg, int len) } /* ---------- + * pgstat_recv_subworker_xact_end() - + * + * Process a SUBWORKERXACTEND message. + * ---------- + */ +static void +pgstat_recv_subworker_xact_end(PgStat_MsgSubWorkerXactEnd *msg, int len) +{ + PgStat_StatDBEntry *dbentry; + PgStat_StatSubWorkerEntry *wentry; + + dbentry = pgstat_get_db_entry(msg->m_databaseid, true); + wentry = pgstat_get_subworker_entry(dbentry, msg->m_subid, + InvalidOid, true); + Assert(wentry); + + switch (msg->m_command) + { + case LOGICAL_REP_MSG_COMMIT: + case LOGICAL_REP_MSG_STREAM_COMMIT: + case LOGICAL_REP_MSG_COMMIT_PREPARED: + wentry->commit_count++; + break; + case LOGICAL_REP_MSG_ROLLBACK_PREPARED: + wentry->abort_count++; + break; + default: + elog(ERROR, "unexpected logical message type as transaction end"); + break; + } +} + +/* ---------- * pgstat_recv_subworker_error() - * * Process a SUBWORKERERROR message. @@ -6193,6 +6258,20 @@ pgstat_recv_subworker_error(PgStat_MsgSubWorkerError *msg, int len) subwentry->last_error_time = msg->m_timestamp; strlcpy(subwentry->last_error_message, msg->m_message, PGSTAT_SUBWORKERERROR_MSGLEN); + + /* + * If this is a new error reported by table sync worker, consolidate this + * error count into the entry of apply worker. + */ + if (OidIsValid(msg->m_subrelid)) + { + /* Gain the apply worker stats */ + subwentry = pgstat_get_subworker_entry(dbentry, msg->m_subid, + InvalidOid, true); + subwentry->error_count++; + } + else + subwentry->error_count++; /* increment the apply worker's counter. */ } /* ---------- diff --git a/src/backend/replication/logical/tablesync.c b/src/backend/replication/logical/tablesync.c index f07983a..e2bfefd 100644 --- a/src/backend/replication/logical/tablesync.c +++ b/src/backend/replication/logical/tablesync.c @@ -1150,6 +1150,13 @@ copy_table_done: SpinLockRelease(&MyLogicalRepWorker->relmutex); /* + * Report the success of table sync as one commit to consolidate all + * transaction stats into one record. + */ + pgstat_report_subworker_xact_end(MyLogicalRepWorker->subid, + LOGICAL_REP_MSG_COMMIT); + + /* * Finally, wait until the main apply worker tells us to catch up and then * return to let LogicalRepApplyLoop do it. */ diff --git a/src/backend/replication/logical/worker.c b/src/backend/replication/logical/worker.c index 2e79302..d897e63 100644 --- a/src/backend/replication/logical/worker.c +++ b/src/backend/replication/logical/worker.c @@ -815,6 +815,14 @@ apply_handle_commit(StringInfo s) apply_handle_commit_internal(&commit_data); + /* + * Report ealier than the call of process_syncing_tables() not to miss an + * increment of commit_count in case it leads to the process exit. See + * process_syncing_tables_for_apply(). + */ + pgstat_report_subworker_xact_end(MyLogicalRepWorker->subid, + LOGICAL_REP_MSG_COMMIT); + /* Process any tables that are being synchronized in parallel. */ process_syncing_tables(commit_data.end_lsn); @@ -960,6 +968,10 @@ apply_handle_commit_prepared(StringInfo s) store_flush_position(prepare_data.end_lsn); in_remote_transaction = false; + /* Update stats */ + pgstat_report_subworker_xact_end(MyLogicalRepWorker->subid, + LOGICAL_REP_MSG_COMMIT_PREPARED); + /* Process any tables that are being synchronized in parallel. */ process_syncing_tables(prepare_data.end_lsn); @@ -1011,6 +1023,10 @@ apply_handle_rollback_prepared(StringInfo s) store_flush_position(rollback_data.rollback_end_lsn); in_remote_transaction = false; + /* Update stats */ + pgstat_report_subworker_xact_end(MyLogicalRepWorker->subid, + LOGICAL_REP_MSG_ROLLBACK_PREPARED); + /* Process any tables that are being synchronized in parallel. */ process_syncing_tables(rollback_data.rollback_end_lsn); @@ -1435,6 +1451,10 @@ apply_handle_stream_commit(StringInfo s) /* unlink the files with serialized changes and subxact info */ stream_cleanup_files(MyLogicalRepWorker->subid, xid); + /* Update stats */ + pgstat_report_subworker_xact_end(MyLogicalRepWorker->subid, + LOGICAL_REP_MSG_STREAM_COMMIT); + /* Process any tables that are being synchronized in parallel. */ process_syncing_tables(commit_data.end_lsn); diff --git a/src/backend/utils/adt/pgstatfuncs.c b/src/backend/utils/adt/pgstatfuncs.c index f529c15..25a9fd2 100644 --- a/src/backend/utils/adt/pgstatfuncs.c +++ b/src/backend/utils/adt/pgstatfuncs.c @@ -2415,7 +2415,7 @@ pg_stat_get_replication_slot(PG_FUNCTION_ARGS) Datum pg_stat_get_subscription_worker(PG_FUNCTION_ARGS) { -#define PG_STAT_GET_SUBSCRIPTION_WORKER_COLS 8 +#define PG_STAT_GET_SUBSCRIPTION_WORKER_COLS 11 Oid subid = PG_GETARG_OID(0); Oid subrelid; TupleDesc tupdesc; @@ -2442,17 +2442,23 @@ pg_stat_get_subscription_worker(PG_FUNCTION_ARGS) OIDOID, -1, 0); TupleDescInitEntry(tupdesc, (AttrNumber) 2, "subrelid", OIDOID, -1, 0); - TupleDescInitEntry(tupdesc, (AttrNumber) 3, "last_error_relid", + TupleDescInitEntry(tupdesc, (AttrNumber) 3, "commit_count", + INT8OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) 4, "error_count", + INT8OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) 5, "abort_count", + INT8OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) 6, "last_error_relid", OIDOID, -1, 0); - TupleDescInitEntry(tupdesc, (AttrNumber) 4, "last_error_command", + TupleDescInitEntry(tupdesc, (AttrNumber) 7, "last_error_command", TEXTOID, -1, 0); - TupleDescInitEntry(tupdesc, (AttrNumber) 5, "last_error_xid", + TupleDescInitEntry(tupdesc, (AttrNumber) 8, "last_error_xid", XIDOID, -1, 0); - TupleDescInitEntry(tupdesc, (AttrNumber) 6, "last_error_count", + TupleDescInitEntry(tupdesc, (AttrNumber) 9, "last_error_count", INT8OID, -1, 0); - TupleDescInitEntry(tupdesc, (AttrNumber) 7, "last_error_message", + TupleDescInitEntry(tupdesc, (AttrNumber) 10, "last_error_message", TEXTOID, -1, 0); - TupleDescInitEntry(tupdesc, (AttrNumber) 8, "last_error_time", + TupleDescInitEntry(tupdesc, (AttrNumber) 11, "last_error_time", TIMESTAMPTZOID, -1, 0); BlessTupleDesc(tupdesc); @@ -2470,6 +2476,11 @@ pg_stat_get_subscription_worker(PG_FUNCTION_ARGS) else nulls[i++] = true; + /* transaction stats */ + values[i++] = Int64GetDatum(wentry->commit_count); + values[i++] = Int64GetDatum(wentry->error_count); + values[i++] = Int64GetDatum(wentry->abort_count); + /* last_error_relid */ if (OidIsValid(wentry->last_error_relid)) values[i++] = ObjectIdGetDatum(wentry->last_error_relid); diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index 79d787c..e78a4a2 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -5393,9 +5393,9 @@ proname => 'pg_stat_get_subscription_worker', prorows => '1', proisstrict => 'f', proretset => 't', provolatile => 's', proparallel => 'r', prorettype => 'record', proargtypes => 'oid oid', - proallargtypes => '{oid,oid,oid,oid,oid,text,xid,int8,text,timestamptz}', - proargmodes => '{i,i,o,o,o,o,o,o,o,o}', - proargnames => '{subid,subrelid,subid,subrelid,last_error_relid,last_error_command,last_error_xid,last_error_count,last_error_message,last_error_time}', + proallargtypes => '{oid,oid,oid,oid,int8,int8,int8,oid,text,xid,int8,text,timestamptz}', + proargmodes => '{i,i,o,o,o,o,o,o,o,o,o,o,o}', + proargnames => '{subid,subrelid,subid,subrelid,commit_count,error_count,abort_count,last_error_relid,last_error_command,last_error_xid,last_error_count,last_error_message,last_error_time}', prosrc => 'pg_stat_get_subscription_worker' }, { oid => '6118', descr => 'statistics: information about subscription', proname => 'pg_stat_get_subscription', prorows => '10', proisstrict => 'f', diff --git a/src/include/pgstat.h b/src/include/pgstat.h index 5b51b58..94ad151 100644 --- a/src/include/pgstat.h +++ b/src/include/pgstat.h @@ -86,6 +86,7 @@ typedef enum StatMsgType PGSTAT_MTYPE_DISCONNECT, PGSTAT_MTYPE_SUBSCRIPTIONPURGE, PGSTAT_MTYPE_SUBWORKERERROR, + PGSTAT_MTYPE_SUBWORKERXACTEND, } StatMsgType; /* ---------- @@ -558,6 +559,25 @@ typedef struct PgStat_MsgSubscriptionPurge } PgStat_MsgSubscriptionPurge; /* ---------- + * PgStat_MsgSubscriptionXactEnd Sent by the apply worker to report transaction + * ends. + * ---------- + */ +typedef struct PgStat_MsgSubWorkerXactEnd +{ + PgStat_MsgHdr m_hdr; + + /* determine the worker entry */ + Oid m_databaseid; + Oid m_subid; + Oid m_subrelid; + + /* necessary to determine column to increment */ + LogicalRepMsgType m_command; + +} PgStat_MsgSubWorkerXactEnd; + +/* ---------- * PgStat_MsgSubWorkerError Sent by the apply worker or the table sync * worker to report the error occurred while * processing changes. @@ -769,6 +789,7 @@ typedef union PgStat_Msg PgStat_MsgDisconnect msg_disconnect; PgStat_MsgSubscriptionPurge msg_subscriptionpurge; PgStat_MsgSubWorkerError msg_subworkererror; + PgStat_MsgSubWorkerXactEnd msg_subworkerxactend; } PgStat_Msg; @@ -1010,6 +1031,13 @@ typedef struct PgStat_StatSubWorkerEntry PgStat_StatSubWorkerKey key; /* hash key (must be first) */ /* + * Cumulative transaction statistics of subscription worker + */ + PgStat_Counter commit_count; + PgStat_Counter error_count; + PgStat_Counter abort_count; + + /* * Subscription worker error statistics representing an error that * occurred during application of changes or the initial table * synchronization. @@ -1131,6 +1159,7 @@ extern void pgstat_report_checksum_failure(void); extern void pgstat_report_replslot(const PgStat_StatReplSlotEntry *repSlotStat); extern void pgstat_report_replslot_create(const char *slotname); extern void pgstat_report_replslot_drop(const char *slotname); +extern void pgstat_report_subworker_xact_end(Oid subid, LogicalRepMsgType command); extern void pgstat_report_subworker_error(Oid subid, Oid subrelid, Oid relid, LogicalRepMsgType command, TransactionId xid, const char *errmsg); diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out index b58b062..6f62f0d 100644 --- a/src/test/regress/expected/rules.out +++ b/src/test/regress/expected/rules.out @@ -2097,6 +2097,9 @@ pg_stat_subscription| SELECT su.oid AS subid, pg_stat_subscription_workers| SELECT w.subid, s.subname, w.subrelid, + w.commit_count, + w.error_count, + w.abort_count, w.last_error_relid, w.last_error_command, w.last_error_xid, @@ -2110,7 +2113,7 @@ pg_stat_subscription_workers| SELECT w.subid, SELECT pg_subscription_rel.srsubid AS subid, pg_subscription_rel.srrelid AS relid FROM pg_subscription_rel) sr, - (LATERAL pg_stat_get_subscription_worker(sr.subid, sr.relid) w(subid, subrelid, last_error_relid, last_error_command, last_error_xid, last_error_count, last_error_message, last_error_time) + (LATERAL pg_stat_get_subscription_worker(sr.subid, sr.relid) w(subid, subrelid, commit_count, error_count, abort_count, last_error_relid, last_error_command, last_error_xid, last_error_count, last_error_message, last_error_time) JOIN pg_subscription s ON ((w.subid = s.oid))); pg_stat_sys_indexes| SELECT pg_stat_all_indexes.relid, pg_stat_all_indexes.indexrelid, diff --git a/src/test/subscription/t/027_worker_xact_stats.pl b/src/test/subscription/t/027_worker_xact_stats.pl new file mode 100644 index 0000000..31dbea1 --- /dev/null +++ b/src/test/subscription/t/027_worker_xact_stats.pl @@ -0,0 +1,162 @@ + +# Copyright (c) 2021, PostgreSQL Global Development Group + +# Tests for subscription worker statistics during apply. +use strict; +use warnings; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More tests => 1; + +# Create publisher node +my $node_publisher = PostgreSQL::Test::Cluster->new('publisher'); +$node_publisher->init(allows_streaming => 'logical'); +$node_publisher->append_conf( + 'postgresql.conf', qq[ +logical_decoding_work_mem = 64kB +max_prepared_transactions = 10 +max_wal_senders = 10 +wal_sender_timeout = 0 +]); +$node_publisher->start; + +# Create subscriber node +my $node_subscriber = PostgreSQL::Test::Cluster->new('subscriber'); +$node_subscriber->init(allows_streaming => 'logical'); +$node_subscriber->append_conf( + 'postgresql.conf', qq[ +max_prepared_transactions = 10 +]); +$node_subscriber->start; + +# Setup structures on the publisher and the subscriber. +# Insert one record that it will cause duplication error +# during the initial table sync. +$node_publisher->safe_psql( + 'postgres', qq[ +CREATE TABLE test_tab1 (a int); +CREATE TABLE test_tab2 (a int); +INSERT INTO test_tab1 VALUES (0); +INSERT INTO test_tab2 VALUES (0); +]); +$node_subscriber->safe_psql( + 'postgres', qq[ +CREATE TABLE test_tab1 (a int primary key); +CREATE TABLE test_tab2 (a int primary key); +INSERT INTO test_tab2 VALUES (0); +]); + +# Setup logical replication +my $publisher_connstr = $node_publisher->connstr . ' dbname=postgres'; +$node_publisher->safe_psql('postgres', + "CREATE PUBLICATION tap_pub FOR TABLE test_tab1, test_tab2"); + +# There's no entry at the beginning +my $result = $node_subscriber->safe_psql('postgres', + "SELECT count(*) FROM pg_stat_subscription_workers"); +is($result, q(0), 'no entry for transaction stats yet'); + +my $appname = 'tap_sub'; +my $subopts = 'streaming = on, two_phase = on'; +$node_subscriber->safe_psql( + 'postgres', qq[ +CREATE SUBSCRIPTION tap_sub +CONNECTION '$publisher_connstr application_name=$appname' +PUBLICATION tap_pub WITH ($subopts); +]); + +# Table synchronization tests. The commit_count gets +# incremented by the number of successful sync completions. +$node_subscriber->poll_query_until('postgres', + "SELECT count(1) = 1 FROM pg_stat_subscription_workers where commit_count = 1" +) or die "didn't get table sync commit."; + +# Resolve the error after confirming error_count has been incremented. +$node_subscriber->poll_query_until('postgres', + "SELECT count(1) = 1 FROM pg_stat_subscription_workers where error_count > 0;" +) or die "didn't get any error of table sync."; +$node_subscriber->safe_psql('postgres', "DELETE FROM test_tab2"); +$node_subscriber->poll_query_until('postgres', + "SELECT count(1) = 1 FROM pg_stat_subscription_workers where commit_count = 2;" +) or die "didn't get the 2nd commit of table sync."; + +# COMMIT +$node_publisher->safe_psql('postgres', + "BEGIN; INSERT INTO test_tab1 VALUES (1); COMMIT;"); + +$node_subscriber->poll_query_until('postgres', + "SELECT count(1) = 1 FROM pg_stat_subscription_workers WHERE commit_count = 3" +) or die "didn't get updates of xact stats by commit"; + +# PREPARE & COMMIT PREPARED +$node_publisher->safe_psql('postgres', + "BEGIN; INSERT INTO test_tab1 VALUES (2); PREPARE TRANSACTION 'gid1'"); +$node_publisher->safe_psql('postgres', "COMMIT PREPARED 'gid1';"); +$node_subscriber->poll_query_until('postgres', + "SELECT count(1) = 1 FROM pg_stat_subscription_workers where commit_count = 4" +) or die "didn't get updates of xact stats by commit prepared"; + +# STREAM COMMIT +$node_publisher->safe_psql('postgres', + "BEGIN; INSERT INTO test_tab1 VALUES(generate_series(1001, 2000)); COMMIT;" +); +$node_subscriber->poll_query_until('postgres', + "SELECT count(1) = 1 FROM pg_stat_subscription_workers where commit_count = 5" +) or die "didn't get updates of xact stats by stream commit"; + +# STREAM PREPARE & COMMIT PREPARED +# This should restore the xact size before the shutdown. +$node_publisher->safe_psql( + 'postgres', q[ +BEGIN; +INSERT INTO test_tab1 VALUES(generate_series(2001, 3000)); +PREPARE TRANSACTION 'gid2'; +]); + +# Commit prepared increments the commit_count. +$node_publisher->safe_psql('postgres', "COMMIT PREPARED 'gid2'"); +$node_subscriber->poll_query_until('postgres', + "SELECT count(1) = 1 FROM pg_stat_subscription_workers where commit_count = 6" + ) + or die + "didn't get updates of xact stats by stream prepare and commit prepared"; + +# ROLLBACK PREPARED +$node_publisher->safe_psql( + 'postgres', q[ +BEGIN; +INSERT INTO test_tab1 VALUES (4); +PREPARE TRANSACTION 'gid3'; +]); +$node_publisher->safe_psql('postgres', "ROLLBACK PREPARED 'gid3'"); +$node_subscriber->poll_query_until('postgres', + "SELECT count(1) = 1 FROM pg_stat_subscription_workers where abort_count = 1" +) or die "didn't get updates of xact stats by rollback prepared"; + +# Test of error stats by duplication error. +# Obtain the current error_count that was increased by table sync error +# and compare it with the result after making some errors of apply. +my $prev_err_count = $node_subscriber->safe_psql('postgres', + "SELECT error_count FROM pg_stat_subscription_workers WHERE subrelid IS NULL" +); + +$node_publisher->safe_psql( + 'postgres', q[ +BEGIN; +INSERT INTO test_tab1 VALUES (1); +COMMIT; +]); + +$node_subscriber->poll_query_until('postgres', + "SELECT count(1) = 1 FROM pg_stat_subscription_workers WHERE subrelid IS NULL AND error_count > $prev_err_count" +) or die "didn't get updates of xact stats by error"; + +# Check the cleanup of DROP SUBSCRIPTION +$node_subscriber->safe_psql('postgres', "DROP SUBSCRIPTION tap_sub"); + +$node_subscriber->poll_query_until('postgres', + "SELECT count(1) = 0 FROM pg_stat_subscription_workers") + or die "drop subscription failed to clean up the worker stats"; + +$node_subscriber->stop('fast'); +$node_publisher->stop('fast'); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f41ef0d..2d195ae 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -1945,6 +1945,7 @@ PgStat_MsgResetslrucounter PgStat_MsgSLRU PgStat_MsgSubscriptionPurge PgStat_MsgSubWorkerError +PgStat_MsgSubWorkerXactEnd PgStat_MsgTabpurge PgStat_MsgTabstat PgStat_MsgTempFile -- 1.8.3.1 ^ permalink raw reply [nested|flat] 113+ messages in thread
* Re: Failed transaction statistics to measure the logical replication progress @ 2021-12-06 14:27 vignesh C <[email protected]> parent: [email protected] <[email protected]> 0 siblings, 1 reply; 113+ messages in thread From: vignesh C @ 2021-12-06 14:27 UTC (permalink / raw) To: [email protected] <[email protected]>; +Cc: Masahiko Sawada <[email protected]>; Amit Kapila <[email protected]>; Greg Nancarrow <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]> On Sat, Dec 4, 2021 at 6:32 PM [email protected] <[email protected]> wrote: > > On Friday, December 3, 2021 3:12 PM vignesh C <[email protected]> wrote: > > Thanks for the updated patch. > > Currently we are storing the commit count, error_count and abort_count for > > each table of the table sync operation. If we have thousands of tables, we will > > be storing the information for each of the tables. > > Shouldn't we be storing the consolidated information in this case. > > diff --git a/src/backend/replication/logical/tablesync.c > > b/src/backend/replication/logical/tablesync.c > > index f07983a..02e9486 100644 > > --- a/src/backend/replication/logical/tablesync.c > > +++ b/src/backend/replication/logical/tablesync.c > > @@ -1149,6 +1149,11 @@ copy_table_done: > > MyLogicalRepWorker->relstate_lsn = *origin_startpos; > > SpinLockRelease(&MyLogicalRepWorker->relmutex); > > > > + /* Report the success of table sync. */ > > + pgstat_report_subworker_xact_end(MyLogicalRepWorker->subid, > > + > > MyLogicalRepWorker->relid, > > + > > 0 /* no logical message type */ ); > Okay. > > I united all stats into that of apply worker. > In line with this change, I fixed the TAP tests as well > to cover the updates of stats done by table sync workers. > > Also, during my self-review, I noticed that > I should call pgstat_report_subworker_xact_end() before > process_syncing_tables() because it can lead to process > exit, which results in missing one increment of the stats columns. > I noted this point in a comment as well. Thanks for the updated patch, few comments: 1) We can keep the documentation similar to mention the count includes both table sync worker / main apply worker in case of commit_count/error_count and abort_count to keep it consistent. + <structfield>commit_count</structfield> <type>bigint</type> + </para> + <para> + Number of transactions successfully applied in this subscription. + COMMIT and COMMIT PREPARED increments this counter. + </para></entry> + </row> + + <row> + <entry role="catalog_table_entry"><para role="column_definition"> + <structfield>error_count</structfield> <type>bigint</type> + </para> + <para> + Number of transactions that failed to be applied by the table + sync worker or main apply worker in this subscription. + </para></entry> + </row> + + <row> + <entry role="catalog_table_entry"><para role="column_definition"> + <structfield>abort_count</structfield> <type>bigint</type> + </para> + <para> + Number of transactions aborted in this subscription. + ROLLBACK PREPARED increments this counter. + </para></entry> + </row> 2) Can this be changed: + /* + * If this is a new error reported by table sync worker, consolidate this + * error count into the entry of apply worker. + */ + if (OidIsValid(msg->m_subrelid)) + { + /* Gain the apply worker stats */ + subwentry = pgstat_get_subworker_entry(dbentry, msg->m_subid, + InvalidOid, true); + subwentry->error_count++; + } + else + subwentry->error_count++; /* increment the apply worker's counter. */ To: + /* + * If this is a new error reported by table sync worker, consolidate this + * error count into the entry of apply worker. + */ + if (OidIsValid(msg->m_subrelid)) + /* Gain the apply worker stats */ + subwentry = pgstat_get_subworker_entry(dbentry, msg->m_subid, + InvalidOid, true); + + subwentry->error_count++; /* increment the apply worker's counter. */ 3) Since both 026_worker_stats and 027_worker_xact_stats.pl are testing pg_stat_subscription_workers, can we move the tests to 026_worker_stats.pl. If possible the error_count validation can be combined with the existing tests. diff --git a/src/test/subscription/t/027_worker_xact_stats.pl b/src/test/subscription/t/027_worker_xact_stats.pl new file mode 100644 index 0000000..31dbea1 --- /dev/null +++ b/src/test/subscription/t/027_worker_xact_stats.pl @@ -0,0 +1,162 @@ + +# Copyright (c) 2021, PostgreSQL Global Development Group + +# Tests for subscription worker statistics during apply. +use strict; +use warnings; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More tests => 1; + +# Create publisher node Regards, Vignesh ^ permalink raw reply [nested|flat] 113+ messages in thread
* RE: Failed transaction statistics to measure the logical replication progress @ 2021-12-07 09:42 [email protected] <[email protected]> parent: vignesh C <[email protected]> 0 siblings, 2 replies; 113+ messages in thread From: [email protected] @ 2021-12-07 09:42 UTC (permalink / raw) To: 'vignesh C' <[email protected]>; +Cc: Masahiko Sawada <[email protected]>; Amit Kapila <[email protected]>; Greg Nancarrow <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]> On Monday, December 6, 2021 11:27 PM vignesh C <[email protected]> wrote: > Thanks for the updated patch, few comments: Thank you for your review ! > 1) We can keep the documentation similar to mention the count includes both > table sync worker / main apply worker in case of commit_count/error_count > and abort_count to keep it consistent. > + <structfield>commit_count</structfield> <type>bigint</type> > + </para> > + <para> > + Number of transactions successfully applied in this subscription. > + COMMIT and COMMIT PREPARED increments this counter. > + </para></entry> > + </row> > + > + <row> > + <entry role="catalog_table_entry"><para role="column_definition"> > + <structfield>error_count</structfield> <type>bigint</type> > + </para> > + <para> > + Number of transactions that failed to be applied by the table > + sync worker or main apply worker in this subscription. > + </para></entry> > + </row> > + > + <row> > + <entry role="catalog_table_entry"><para role="column_definition"> > + <structfield>abort_count</structfield> <type>bigint</type> > + </para> > + <para> > + Number of transactions aborted in this subscription. > + ROLLBACK PREPARED increments this counter. > + </para></entry> > + </row> Yeah, you are right. Fixed. Note that abort_count is not used by table sync worker. > 2) Can this be changed: > + /* > + * If this is a new error reported by table sync worker, > consolidate this > + * error count into the entry of apply worker. > + */ > + if (OidIsValid(msg->m_subrelid)) > + { > + /* Gain the apply worker stats */ > + subwentry = pgstat_get_subworker_entry(dbentry, > + msg->m_subid, > + > InvalidOid, true); > + subwentry->error_count++; > + } > + else > + subwentry->error_count++; /* increment the apply > worker's counter. */ > To: > + /* > + * If this is a new error reported by table sync worker, > consolidate this > + * error count into the entry of apply worker. > + */ > + if (OidIsValid(msg->m_subrelid)) > + /* Gain the apply worker stats */ > + subwentry = pgstat_get_subworker_entry(dbentry, > + msg->m_subid, > + > InvalidOid, true); > + > + subwentry->error_count++; /* increment the apply > worker's counter. */ Your suggestion looks better. Also, I fixed some comments of this part so that we don't need to add a separate comment at the bottom for the increment of the apply worker. > 3) Since both 026_worker_stats and 027_worker_xact_stats.pl are testing > pg_stat_subscription_workers, can we move the tests to 026_worker_stats.pl. > If possible the error_count validation can be combined with the existing tests. > diff --git a/src/test/subscription/t/027_worker_xact_stats.pl > b/src/test/subscription/t/027_worker_xact_stats.pl > new file mode 100644 > index 0000000..31dbea1 > --- /dev/null > +++ b/src/test/subscription/t/027_worker_xact_stats.pl > @@ -0,0 +1,162 @@ > + > +# Copyright (c) 2021, PostgreSQL Global Development Group > + > +# Tests for subscription worker statistics during apply. > +use strict; > +use warnings; > +use PostgreSQL::Test::Cluster; > +use PostgreSQL::Test::Utils; > +use Test::More tests => 1; > + > +# Create publisher node Right. I've integrated my tests with 026_worker_stats.pl. I think error_count validations are combined as you suggested. Another change I did is to introduce one function to contribute to better readability of the stats tests. Here, the 026_worker_stats.pl didn't look aligned by pgperltidy. This is not a serious issue at all. Yet, when I ran pgperltidy, the existing codes that required adjustments came into my patch. Therefore, I made a separate part for this. Best Regards, Takamichi Osumi Attachments: [application/octet-stream] v16-0001-Fix-alignments-of-TAP-tests-for-pg_stat_subscrip.patch (7.4K, ../../TYCPR01MB8373C2D4D105B83C378511CAED6E9@TYCPR01MB8373.jpnprd01.prod.outlook.com/2-v16-0001-Fix-alignments-of-TAP-tests-for-pg_stat_subscrip.patch) download | inline diff: From 47d4e900f3d01710c7b769042b2d89ba84eef344 Mon Sep 17 00:00:00 2001 From: Takamichi Osumi <[email protected]> Date: Tue, 7 Dec 2021 07:39:58 +0000 Subject: [PATCH v16 1/2] Fix alignments of TAP tests for pg_stat_subscription_workers --- src/test/subscription/t/026_worker_stats.pl | 129 ++++++++++++++-------------- 1 file changed, 63 insertions(+), 66 deletions(-) diff --git a/src/test/subscription/t/026_worker_stats.pl b/src/test/subscription/t/026_worker_stats.pl index b662be3..8005c54 100644 --- a/src/test/subscription/t/026_worker_stats.pl +++ b/src/test/subscription/t/026_worker_stats.pl @@ -11,33 +11,37 @@ use Test::More tests => 3; # Test if the error reported on pg_stat_subscription_workers view is expected. sub test_subscription_error { - my ($node, $relname, $command, $xid, $by_apply_worker, $errmsg_prefix, $msg) - = @_; + my ($node, $relname, $command, $xid, $by_apply_worker, $errmsg_prefix, + $msg) + = @_; - my $check_sql = qq[ + my $check_sql = qq[ SELECT count(1) > 0 FROM pg_stat_subscription_workers WHERE last_error_relid = '$relname'::regclass AND starts_with(last_error_message, '$errmsg_prefix')]; - # subrelid - $check_sql .= $by_apply_worker - ? qq[ AND subrelid IS NULL] - : qq[ AND subrelid = '$relname'::regclass]; - - # last_error_command - $check_sql .= $command eq '' - ? qq[ AND last_error_command IS NULL] - : qq[ AND last_error_command = '$command']; - - # last_error_xid - $check_sql .= $xid eq '' - ? qq[ AND last_error_xid IS NULL] - : qq[ AND last_error_xid = '$xid'::xid]; - - # Wait for the particular error statistics to be reported. - $node->poll_query_until('postgres', $check_sql, -) or die "Timed out while waiting for " . $msg; + # subrelid + $check_sql .= + $by_apply_worker + ? qq[ AND subrelid IS NULL] + : qq[ AND subrelid = '$relname'::regclass]; + + # last_error_command + $check_sql .= + $command eq '' + ? qq[ AND last_error_command IS NULL] + : qq[ AND last_error_command = '$command']; + + # last_error_xid + $check_sql .= + $xid eq '' + ? qq[ AND last_error_xid IS NULL] + : qq[ AND last_error_xid = '$xid'::xid]; + + # Wait for the particular error statistics to be reported. + $node->poll_query_until('postgres', $check_sql,) + or die "Timed out while waiting for " . $msg; } # Create publisher node. @@ -51,8 +55,9 @@ $node_subscriber->init(allows_streaming => 'logical'); # The subscriber will enter an infinite error loop, so we don't want # to overflow the server log with error messages. -$node_subscriber->append_conf('postgresql.conf', - qq[ +$node_subscriber->append_conf( + 'postgresql.conf', + qq[ wal_retrieve_retry_interval = 2s ]); $node_subscriber->start; @@ -61,8 +66,8 @@ $node_subscriber->start; # create the same tables but with primary keys. Also, insert some data that # will conflict with the data replicated from publisher later. $node_publisher->safe_psql( - 'postgres', - qq[ + 'postgres', + qq[ BEGIN; CREATE TABLE test_tab1 (a int); CREATE TABLE test_tab2 (a int); @@ -71,8 +76,8 @@ INSERT INTO test_tab2 VALUES (1); COMMIT; ]); $node_subscriber->safe_psql( - 'postgres', - qq[ + 'postgres', + qq[ BEGIN; CREATE TABLE test_tab1 (a int primary key); CREATE TABLE test_tab2 (a int primary key); @@ -82,81 +87,73 @@ COMMIT; # Setup publications. my $publisher_connstr = $node_publisher->connstr . ' dbname=postgres'; -$node_publisher->safe_psql( - 'postgres', - "CREATE PUBLICATION tap_pub FOR TABLE test_tab1, test_tab2;"); +$node_publisher->safe_psql('postgres', + "CREATE PUBLICATION tap_pub FOR TABLE test_tab1, test_tab2;"); # There shouldn't be any subscription errors before starting logical replication. -my $result = $node_subscriber->safe_psql( - 'postgres', - "SELECT count(1) FROM pg_stat_subscription_workers"); +my $result = $node_subscriber->safe_psql('postgres', + "SELECT count(1) FROM pg_stat_subscription_workers"); is($result, qq(0), 'check no subscription error'); # Create subscription. The table sync for test_tab2 on tap_sub will enter into # infinite error loop due to violating the unique constraint. -$node_subscriber->safe_psql( - 'postgres', - "CREATE SUBSCRIPTION tap_sub CONNECTION '$publisher_connstr' PUBLICATION tap_pub;"); +$node_subscriber->safe_psql('postgres', + "CREATE SUBSCRIPTION tap_sub CONNECTION '$publisher_connstr' PUBLICATION tap_pub;" +); $node_publisher->wait_for_catchup('tap_sub'); # Wait for initial table sync for test_tab1 to finish. $node_subscriber->poll_query_until( - 'postgres', - qq[ + 'postgres', + qq[ SELECT count(1) = 1 FROM pg_subscription_rel WHERE srrelid = 'test_tab1'::regclass AND srsubstate in ('r', 's') ]) or die "Timed out while waiting for subscriber to synchronize data"; # Check the initial data. -$result = $node_subscriber->safe_psql( - 'postgres', - "SELECT count(a) FROM test_tab1"); +$result = + $node_subscriber->safe_psql('postgres', "SELECT count(a) FROM test_tab1"); is($result, q(1), 'check initial data are copied to subscriber'); # Insert more data to test_tab1, raising an error on the subscriber due to # violation of the unique constraint on test_tab1. my $xid = $node_publisher->safe_psql( - 'postgres', - qq[ + 'postgres', + qq[ BEGIN; INSERT INTO test_tab1 VALUES (1); SELECT pg_current_xact_id()::xid; COMMIT; ]); -test_subscription_error($node_subscriber, 'test_tab1', 'INSERT', $xid, - 1, # check apply worker error - qq(duplicate key value violates unique constraint), - 'error reported by the apply worker'); +test_subscription_error( + $node_subscriber, 'test_tab1', 'INSERT', $xid, + 1, # check apply worker error + qq(duplicate key value violates unique constraint), + 'error reported by the apply worker'); # Check the table sync worker's error in the view. -test_subscription_error($node_subscriber, 'test_tab2', '', '', - 0, # check tablesync worker error - qq(duplicate key value violates unique constraint), - 'the error reported by the table sync worker'); +test_subscription_error( + $node_subscriber, 'test_tab2', '', '', + 0, # check tablesync worker error + qq(duplicate key value violates unique constraint), + 'the error reported by the table sync worker'); # Test for resetting subscription worker statistics. # Truncate test_tab1 and test_tab2 so that applying changes and table sync can # continue, respectively. -$node_subscriber->safe_psql( - 'postgres', - "TRUNCATE test_tab1, test_tab2;"); +$node_subscriber->safe_psql('postgres', "TRUNCATE test_tab1, test_tab2;"); # Wait for the data to be replicated. -$node_subscriber->poll_query_until( - 'postgres', - "SELECT count(1) > 0 FROM test_tab1"); -$node_subscriber->poll_query_until( - 'postgres', - "SELECT count(1) > 0 FROM test_tab2"); +$node_subscriber->poll_query_until('postgres', + "SELECT count(1) > 0 FROM test_tab1"); +$node_subscriber->poll_query_until('postgres', + "SELECT count(1) > 0 FROM test_tab2"); # There shouldn't be any errors in the view after dropping the subscription. -$node_subscriber->safe_psql( - 'postgres', - "DROP SUBSCRIPTION tap_sub;"); -$result = $node_subscriber->safe_psql( - 'postgres', - "SELECT count(1) FROM pg_stat_subscription_workers"); +$node_subscriber->safe_psql('postgres', "DROP SUBSCRIPTION tap_sub;"); +$result = $node_subscriber->safe_psql('postgres', + "SELECT count(1) FROM pg_stat_subscription_workers"); is($result, q(0), 'no error after dropping subscription'); $node_subscriber->stop('fast'); -- 1.8.3.1 [application/octet-stream] v16-0002-Extend-pg_stat_subscription_workers-to-include-g.patch (24.7K, ../../TYCPR01MB8373C2D4D105B83C378511CAED6E9@TYCPR01MB8373.jpnprd01.prod.outlook.com/3-v16-0002-Extend-pg_stat_subscription_workers-to-include-g.patch) download | inline diff: From 3028e49dd656fc820c32ffc725b55a9201898291 Mon Sep 17 00:00:00 2001 From: Takamichi Osumi <[email protected]> Date: Tue, 7 Dec 2021 08:53:47 +0000 Subject: [PATCH v16 2/2] Extend pg_stat_subscription_workers to include general transaction statistics Categorize transactions of logical replication subscriber into three types (commit, error, abort) and introduce cumulative columns of those numbers. The stats for table sync worker is merged into those of the apply worker. At present, there's no special consideration to statistics of spool file logic such as amount of data spooled to disk, its count or corresponding statistics associated with STREAM ABORT. However, those can be added later. Author: Takamichi Osumi Reviewed-by: Amit Kapila, Masahiko Sawada, Hou Zhijie, Greg Nancarrow, Vignesh C, Ajin Cherian Discussion: https://www.postgresql.org/message-id/OSBPR01MB48887CA8F40C8D984A6DC00CED199%40OSBPR01MB4888.jpnprd01.prod.outlook.com --- doc/src/sgml/monitoring.sgml | 43 +++++++++- src/backend/catalog/system_views.sql | 3 + src/backend/postmaster/pgstat.c | 75 ++++++++++++++++++ src/backend/replication/logical/tablesync.c | 7 ++ src/backend/replication/logical/worker.c | 20 +++++ src/backend/utils/adt/pgstatfuncs.c | 25 ++++-- src/include/catalog/pg_proc.dat | 6 +- src/include/pgstat.h | 29 +++++++ src/test/regress/expected/rules.out | 5 +- src/test/subscription/t/026_worker_stats.pl | 118 +++++++++++++++++++++++++++- src/tools/pgindent/typedefs.list | 1 + 11 files changed, 313 insertions(+), 19 deletions(-) diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml index 62f2a33..1feb075 100644 --- a/doc/src/sgml/monitoring.sgml +++ b/doc/src/sgml/monitoring.sgml @@ -629,8 +629,8 @@ postgres 27093 0.0 0.0 30096 2752 ? Ss 11:34 0:00 postgres: ser <row> <entry><structname>pg_stat_subscription_workers</structname><indexterm><primary>pg_stat_subscription_workers</primary></indexterm></entry> - <entry>One row per subscription worker, showing statistics about errors - that occurred on that subscription worker. + <entry>One row per subscription worker, showing statistics about transactions + and errors that occurred on that subscription worker. See <link linkend="monitoring-pg-stat-subscription-workers"> <structname>pg_stat_subscription_workers</structname></link> for details. </entry> @@ -3074,8 +3074,10 @@ SELECT pid, wait_event_type, wait_event FROM pg_stat_activity WHERE wait_event i The <structname>pg_stat_subscription_workers</structname> view will contain one row per subscription worker on which errors have occurred, for workers applying logical replication changes and workers handling the initial data - copy of the subscribed tables. The statistics entry is removed when the - corresponding subscription is dropped. + copy of the subscribed tables. Also, the row corresponding to the apply + worker shows all transaction statistics of both types of workers on the + subscription. The statistics entry is removed when the corresponding + subscription is dropped. </para> <table id="pg-stat-subscription-workers" xreflabel="pg_stat_subscription_workers"> @@ -3123,6 +3125,39 @@ SELECT pid, wait_event_type, wait_event FROM pg_stat_activity WHERE wait_event i <row> <entry role="catalog_table_entry"><para role="column_definition"> + <structfield>commit_count</structfield> <type>bigint</type> + </para> + <para> + Number of transactions successfully applied in this subscription. + COMMIT, COMMIT PREPARED and successful table copy of table sync + worker increment this counter. + </para></entry> + </row> + + <row> + <entry role="catalog_table_entry"><para role="column_definition"> + <structfield>error_count</structfield> <type>bigint</type> + </para> + <para> + Number of transactions that failed to be applied by the table + sync worker or main apply worker in this subscription. This + counter is updated after confirming the error is not same as + the previous one. + </para></entry> + </row> + + <row> + <entry role="catalog_table_entry"><para role="column_definition"> + <structfield>abort_count</structfield> <type>bigint</type> + </para> + <para> + Number of transactions aborted in this subscription. + ROLLBACK PREPARED increments this counter. + </para></entry> + </row> + + <row> + <entry role="catalog_table_entry"><para role="column_definition"> <structfield>last_error_relid</structfield> <type>oid</type> </para> <para> diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index 61b515c..d2ca10a 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1267,6 +1267,9 @@ CREATE VIEW pg_stat_subscription_workers AS w.subid, s.subname, w.subrelid, + w.commit_count, + w.error_count, + w.abort_count, w.last_error_relid, w.last_error_command, w.last_error_xid, diff --git a/src/backend/postmaster/pgstat.c b/src/backend/postmaster/pgstat.c index 7264d2c..fef9181 100644 --- a/src/backend/postmaster/pgstat.c +++ b/src/backend/postmaster/pgstat.c @@ -382,6 +382,7 @@ static void pgstat_recv_replslot(PgStat_MsgReplSlot *msg, int len); static void pgstat_recv_tempfile(PgStat_MsgTempFile *msg, int len); static void pgstat_recv_subscription_purge(PgStat_MsgSubscriptionPurge *msg, int len); static void pgstat_recv_subworker_error(PgStat_MsgSubWorkerError *msg, int len); +static void pgstat_recv_subworker_xact_end(PgStat_MsgSubWorkerXactEnd *msg, int len); /* ------------------------------------------------------------ * Public functions called from postmaster follow @@ -1949,6 +1950,30 @@ pgstat_report_replslot_drop(const char *slotname) } /* ---------- + * pgstat_report_subworker_xact_end() - + * + * Tell the collector that worker transaction has successfully completed. + * ---------- + */ +void +pgstat_report_subworker_xact_end(Oid subid, LogicalRepMsgType command) +{ + PgStat_MsgSubWorkerXactEnd msg; + + /* LOGICAL_REP_MSG_COMMIT cares about table sync's commit too */ + Assert(command == LOGICAL_REP_MSG_COMMIT || + command == LOGICAL_REP_MSG_STREAM_COMMIT || + command == LOGICAL_REP_MSG_COMMIT_PREPARED || + command == LOGICAL_REP_MSG_ROLLBACK_PREPARED); + + pgstat_setheader(&msg.m_hdr, PGSTAT_MTYPE_SUBWORKERXACTEND); + msg.m_databaseid = MyDatabaseId; + msg.m_subid = subid; + msg.m_command = command; + pgstat_send(&msg, sizeof(PgStat_MsgSubWorkerXactEnd)); +} + +/* ---------- * pgstat_report_subworker_error() - * * Tell the collector about the subscription worker error. @@ -3746,6 +3771,10 @@ PgstatCollectorMain(int argc, char *argv[]) pgstat_recv_subworker_error(&msg.msg_subworkererror, len); break; + case PGSTAT_MTYPE_SUBWORKERXACTEND: + pgstat_recv_subworker_xact_end(&msg.msg_subworkerxactend, len); + break; + default: break; } @@ -3965,6 +3994,9 @@ pgstat_get_subworker_entry(PgStat_StatDBEntry *dbentry, Oid subid, Oid subrelid, /* If not found, initialize the new one */ if (!found) { + subwentry->commit_count = 0; + subwentry->error_count = 0; + subwentry->abort_count = 0; subwentry->last_error_relid = InvalidOid; subwentry->last_error_command = 0; subwentry->last_error_xid = InvalidTransactionId; @@ -6153,6 +6185,39 @@ pgstat_recv_subscription_purge(PgStat_MsgSubscriptionPurge *msg, int len) } /* ---------- + * pgstat_recv_subworker_xact_end() - + * + * Process a SUBWORKERXACTEND message. + * ---------- + */ +static void +pgstat_recv_subworker_xact_end(PgStat_MsgSubWorkerXactEnd *msg, int len) +{ + PgStat_StatDBEntry *dbentry; + PgStat_StatSubWorkerEntry *wentry; + + dbentry = pgstat_get_db_entry(msg->m_databaseid, true); + wentry = pgstat_get_subworker_entry(dbentry, msg->m_subid, + InvalidOid, true); + Assert(wentry); + + switch (msg->m_command) + { + case LOGICAL_REP_MSG_COMMIT: + case LOGICAL_REP_MSG_STREAM_COMMIT: + case LOGICAL_REP_MSG_COMMIT_PREPARED: + wentry->commit_count++; + break; + case LOGICAL_REP_MSG_ROLLBACK_PREPARED: + wentry->abort_count++; + break; + default: + elog(ERROR, "unexpected logical message type as transaction end"); + break; + } +} + +/* ---------- * pgstat_recv_subworker_error() - * * Process a SUBWORKERERROR message. @@ -6193,6 +6258,16 @@ pgstat_recv_subworker_error(PgStat_MsgSubWorkerError *msg, int len) subwentry->last_error_time = msg->m_timestamp; strlcpy(subwentry->last_error_message, msg->m_message, PGSTAT_SUBWORKERERROR_MSGLEN); + + /* + * If this is a new error reported by table sync worker, consolidate this + * error count into the entry of apply worker, by swapping the stats + * entries. + */ + if (OidIsValid(msg->m_subrelid)) + subwentry = pgstat_get_subworker_entry(dbentry, msg->m_subid, + InvalidOid, true); + subwentry->error_count++; } /* ---------- diff --git a/src/backend/replication/logical/tablesync.c b/src/backend/replication/logical/tablesync.c index f07983a..e2bfefd 100644 --- a/src/backend/replication/logical/tablesync.c +++ b/src/backend/replication/logical/tablesync.c @@ -1150,6 +1150,13 @@ copy_table_done: SpinLockRelease(&MyLogicalRepWorker->relmutex); /* + * Report the success of table sync as one commit to consolidate all + * transaction stats into one record. + */ + pgstat_report_subworker_xact_end(MyLogicalRepWorker->subid, + LOGICAL_REP_MSG_COMMIT); + + /* * Finally, wait until the main apply worker tells us to catch up and then * return to let LogicalRepApplyLoop do it. */ diff --git a/src/backend/replication/logical/worker.c b/src/backend/replication/logical/worker.c index 2e79302..d897e63 100644 --- a/src/backend/replication/logical/worker.c +++ b/src/backend/replication/logical/worker.c @@ -815,6 +815,14 @@ apply_handle_commit(StringInfo s) apply_handle_commit_internal(&commit_data); + /* + * Report ealier than the call of process_syncing_tables() not to miss an + * increment of commit_count in case it leads to the process exit. See + * process_syncing_tables_for_apply(). + */ + pgstat_report_subworker_xact_end(MyLogicalRepWorker->subid, + LOGICAL_REP_MSG_COMMIT); + /* Process any tables that are being synchronized in parallel. */ process_syncing_tables(commit_data.end_lsn); @@ -960,6 +968,10 @@ apply_handle_commit_prepared(StringInfo s) store_flush_position(prepare_data.end_lsn); in_remote_transaction = false; + /* Update stats */ + pgstat_report_subworker_xact_end(MyLogicalRepWorker->subid, + LOGICAL_REP_MSG_COMMIT_PREPARED); + /* Process any tables that are being synchronized in parallel. */ process_syncing_tables(prepare_data.end_lsn); @@ -1011,6 +1023,10 @@ apply_handle_rollback_prepared(StringInfo s) store_flush_position(rollback_data.rollback_end_lsn); in_remote_transaction = false; + /* Update stats */ + pgstat_report_subworker_xact_end(MyLogicalRepWorker->subid, + LOGICAL_REP_MSG_ROLLBACK_PREPARED); + /* Process any tables that are being synchronized in parallel. */ process_syncing_tables(rollback_data.rollback_end_lsn); @@ -1435,6 +1451,10 @@ apply_handle_stream_commit(StringInfo s) /* unlink the files with serialized changes and subxact info */ stream_cleanup_files(MyLogicalRepWorker->subid, xid); + /* Update stats */ + pgstat_report_subworker_xact_end(MyLogicalRepWorker->subid, + LOGICAL_REP_MSG_STREAM_COMMIT); + /* Process any tables that are being synchronized in parallel. */ process_syncing_tables(commit_data.end_lsn); diff --git a/src/backend/utils/adt/pgstatfuncs.c b/src/backend/utils/adt/pgstatfuncs.c index f529c15..25a9fd2 100644 --- a/src/backend/utils/adt/pgstatfuncs.c +++ b/src/backend/utils/adt/pgstatfuncs.c @@ -2415,7 +2415,7 @@ pg_stat_get_replication_slot(PG_FUNCTION_ARGS) Datum pg_stat_get_subscription_worker(PG_FUNCTION_ARGS) { -#define PG_STAT_GET_SUBSCRIPTION_WORKER_COLS 8 +#define PG_STAT_GET_SUBSCRIPTION_WORKER_COLS 11 Oid subid = PG_GETARG_OID(0); Oid subrelid; TupleDesc tupdesc; @@ -2442,17 +2442,23 @@ pg_stat_get_subscription_worker(PG_FUNCTION_ARGS) OIDOID, -1, 0); TupleDescInitEntry(tupdesc, (AttrNumber) 2, "subrelid", OIDOID, -1, 0); - TupleDescInitEntry(tupdesc, (AttrNumber) 3, "last_error_relid", + TupleDescInitEntry(tupdesc, (AttrNumber) 3, "commit_count", + INT8OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) 4, "error_count", + INT8OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) 5, "abort_count", + INT8OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) 6, "last_error_relid", OIDOID, -1, 0); - TupleDescInitEntry(tupdesc, (AttrNumber) 4, "last_error_command", + TupleDescInitEntry(tupdesc, (AttrNumber) 7, "last_error_command", TEXTOID, -1, 0); - TupleDescInitEntry(tupdesc, (AttrNumber) 5, "last_error_xid", + TupleDescInitEntry(tupdesc, (AttrNumber) 8, "last_error_xid", XIDOID, -1, 0); - TupleDescInitEntry(tupdesc, (AttrNumber) 6, "last_error_count", + TupleDescInitEntry(tupdesc, (AttrNumber) 9, "last_error_count", INT8OID, -1, 0); - TupleDescInitEntry(tupdesc, (AttrNumber) 7, "last_error_message", + TupleDescInitEntry(tupdesc, (AttrNumber) 10, "last_error_message", TEXTOID, -1, 0); - TupleDescInitEntry(tupdesc, (AttrNumber) 8, "last_error_time", + TupleDescInitEntry(tupdesc, (AttrNumber) 11, "last_error_time", TIMESTAMPTZOID, -1, 0); BlessTupleDesc(tupdesc); @@ -2470,6 +2476,11 @@ pg_stat_get_subscription_worker(PG_FUNCTION_ARGS) else nulls[i++] = true; + /* transaction stats */ + values[i++] = Int64GetDatum(wentry->commit_count); + values[i++] = Int64GetDatum(wentry->error_count); + values[i++] = Int64GetDatum(wentry->abort_count); + /* last_error_relid */ if (OidIsValid(wentry->last_error_relid)) values[i++] = ObjectIdGetDatum(wentry->last_error_relid); diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index 79d787c..e78a4a2 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -5393,9 +5393,9 @@ proname => 'pg_stat_get_subscription_worker', prorows => '1', proisstrict => 'f', proretset => 't', provolatile => 's', proparallel => 'r', prorettype => 'record', proargtypes => 'oid oid', - proallargtypes => '{oid,oid,oid,oid,oid,text,xid,int8,text,timestamptz}', - proargmodes => '{i,i,o,o,o,o,o,o,o,o}', - proargnames => '{subid,subrelid,subid,subrelid,last_error_relid,last_error_command,last_error_xid,last_error_count,last_error_message,last_error_time}', + proallargtypes => '{oid,oid,oid,oid,int8,int8,int8,oid,text,xid,int8,text,timestamptz}', + proargmodes => '{i,i,o,o,o,o,o,o,o,o,o,o,o}', + proargnames => '{subid,subrelid,subid,subrelid,commit_count,error_count,abort_count,last_error_relid,last_error_command,last_error_xid,last_error_count,last_error_message,last_error_time}', prosrc => 'pg_stat_get_subscription_worker' }, { oid => '6118', descr => 'statistics: information about subscription', proname => 'pg_stat_get_subscription', prorows => '10', proisstrict => 'f', diff --git a/src/include/pgstat.h b/src/include/pgstat.h index 5b51b58..94ad151 100644 --- a/src/include/pgstat.h +++ b/src/include/pgstat.h @@ -86,6 +86,7 @@ typedef enum StatMsgType PGSTAT_MTYPE_DISCONNECT, PGSTAT_MTYPE_SUBSCRIPTIONPURGE, PGSTAT_MTYPE_SUBWORKERERROR, + PGSTAT_MTYPE_SUBWORKERXACTEND, } StatMsgType; /* ---------- @@ -558,6 +559,25 @@ typedef struct PgStat_MsgSubscriptionPurge } PgStat_MsgSubscriptionPurge; /* ---------- + * PgStat_MsgSubscriptionXactEnd Sent by the apply worker to report transaction + * ends. + * ---------- + */ +typedef struct PgStat_MsgSubWorkerXactEnd +{ + PgStat_MsgHdr m_hdr; + + /* determine the worker entry */ + Oid m_databaseid; + Oid m_subid; + Oid m_subrelid; + + /* necessary to determine column to increment */ + LogicalRepMsgType m_command; + +} PgStat_MsgSubWorkerXactEnd; + +/* ---------- * PgStat_MsgSubWorkerError Sent by the apply worker or the table sync * worker to report the error occurred while * processing changes. @@ -769,6 +789,7 @@ typedef union PgStat_Msg PgStat_MsgDisconnect msg_disconnect; PgStat_MsgSubscriptionPurge msg_subscriptionpurge; PgStat_MsgSubWorkerError msg_subworkererror; + PgStat_MsgSubWorkerXactEnd msg_subworkerxactend; } PgStat_Msg; @@ -1010,6 +1031,13 @@ typedef struct PgStat_StatSubWorkerEntry PgStat_StatSubWorkerKey key; /* hash key (must be first) */ /* + * Cumulative transaction statistics of subscription worker + */ + PgStat_Counter commit_count; + PgStat_Counter error_count; + PgStat_Counter abort_count; + + /* * Subscription worker error statistics representing an error that * occurred during application of changes or the initial table * synchronization. @@ -1131,6 +1159,7 @@ extern void pgstat_report_checksum_failure(void); extern void pgstat_report_replslot(const PgStat_StatReplSlotEntry *repSlotStat); extern void pgstat_report_replslot_create(const char *slotname); extern void pgstat_report_replslot_drop(const char *slotname); +extern void pgstat_report_subworker_xact_end(Oid subid, LogicalRepMsgType command); extern void pgstat_report_subworker_error(Oid subid, Oid subrelid, Oid relid, LogicalRepMsgType command, TransactionId xid, const char *errmsg); diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out index b58b062..6f62f0d 100644 --- a/src/test/regress/expected/rules.out +++ b/src/test/regress/expected/rules.out @@ -2097,6 +2097,9 @@ pg_stat_subscription| SELECT su.oid AS subid, pg_stat_subscription_workers| SELECT w.subid, s.subname, w.subrelid, + w.commit_count, + w.error_count, + w.abort_count, w.last_error_relid, w.last_error_command, w.last_error_xid, @@ -2110,7 +2113,7 @@ pg_stat_subscription_workers| SELECT w.subid, SELECT pg_subscription_rel.srsubid AS subid, pg_subscription_rel.srrelid AS relid FROM pg_subscription_rel) sr, - (LATERAL pg_stat_get_subscription_worker(sr.subid, sr.relid) w(subid, subrelid, last_error_relid, last_error_command, last_error_xid, last_error_count, last_error_message, last_error_time) + (LATERAL pg_stat_get_subscription_worker(sr.subid, sr.relid) w(subid, subrelid, commit_count, error_count, abort_count, last_error_relid, last_error_command, last_error_xid, last_error_count, last_error_message, last_error_time) JOIN pg_subscription s ON ((w.subid = s.oid))); pg_stat_sys_indexes| SELECT pg_stat_all_indexes.relid, pg_stat_all_indexes.indexrelid, diff --git a/src/test/subscription/t/026_worker_stats.pl b/src/test/subscription/t/026_worker_stats.pl index 8005c54..1a952bd 100644 --- a/src/test/subscription/t/026_worker_stats.pl +++ b/src/test/subscription/t/026_worker_stats.pl @@ -1,7 +1,7 @@ # Copyright (c) 2021, PostgreSQL Global Development Group -# Tests for subscription error stats. +# Tests for subscription stats. use strict; use warnings; use PostgreSQL::Test::Cluster; @@ -44,9 +44,32 @@ WHERE last_error_relid = '$relname'::regclass or die "Timed out while waiting for " . $msg; } +# Test the update of general transaction stats which satisfy +# the expected condition has been detected. +sub confirm_transaction_stats_update +{ + my ($node, $condition, $msg) = @_; + + # Check only the stats of the apply worker + my $sql = qq[ +SELECT count(1) = 1 +FROM pg_stat_subscription_workers +WHERE subrelid IS NULL AND $condition]; + + $node->poll_query_until('postgres', $sql) + or die "Timed out while waiting for " . $msg; +} + # Create publisher node. my $node_publisher = PostgreSQL::Test::Cluster->new('publisher'); $node_publisher->init(allows_streaming => 'logical'); +$node_publisher->append_conf( + 'postgresql.conf', qq[ +logical_decoding_work_mem = 64kB +max_prepared_transactions = 10 +max_wal_senders = 10 +wal_sender_timeout = 0 +]); $node_publisher->start; # Create subscriber node. @@ -58,6 +81,7 @@ $node_subscriber->init(allows_streaming => 'logical'); $node_subscriber->append_conf( 'postgresql.conf', qq[ +max_prepared_transactions = 10 wal_retrieve_retry_interval = 2s ]); $node_subscriber->start; @@ -98,7 +122,7 @@ is($result, qq(0), 'check no subscription error'); # Create subscription. The table sync for test_tab2 on tap_sub will enter into # infinite error loop due to violating the unique constraint. $node_subscriber->safe_psql('postgres', - "CREATE SUBSCRIPTION tap_sub CONNECTION '$publisher_connstr' PUBLICATION tap_pub;" + "CREATE SUBSCRIPTION tap_sub CONNECTION '$publisher_connstr' PUBLICATION tap_pub WITH (streaming = on, two_phase = on);" ); $node_publisher->wait_for_catchup('tap_sub'); @@ -116,6 +140,17 @@ $result = $node_subscriber->safe_psql('postgres', "SELECT count(a) FROM test_tab1"); is($result, q(1), 'check initial data are copied to subscriber'); +# Check the update of stats counters. +confirm_transaction_stats_update( + $node_subscriber, + 'commit_count = 1', + 'the commit_count increment by table sync'); + +confirm_transaction_stats_update( + $node_subscriber, + 'error_count = 1', + 'the error_count increment by table sync'); + # Insert more data to test_tab1, raising an error on the subscriber due to # violation of the unique constraint on test_tab1. my $xid = $node_publisher->safe_psql( @@ -132,6 +167,11 @@ test_subscription_error( qq(duplicate key value violates unique constraint), 'error reported by the apply worker'); +confirm_transaction_stats_update( + $node_subscriber, + 'error_count = 2', + 'the error_count increment by the apply worker'); + # Check the table sync worker's error in the view. test_subscription_error( $node_subscriber, 'test_tab2', '', '', @@ -150,11 +190,81 @@ $node_subscriber->poll_query_until('postgres', $node_subscriber->poll_query_until('postgres', "SELECT count(1) > 0 FROM test_tab2"); -# There shouldn't be any errors in the view after dropping the subscription. +confirm_transaction_stats_update( + $node_subscriber, + 'commit_count = 3', + 'the commit_count increments by both workers'); + +# Some more tests for transaction stats +# PREPARE & COMMIT PREPARED +$node_publisher->safe_psql( + 'postgres', qq[ +BEGIN; +INSERT INTO test_tab1 VALUES (2); +PREPARE TRANSACTION 'gid1' +]); +$node_publisher->safe_psql('postgres', "COMMIT PREPARED 'gid1'"); +confirm_transaction_stats_update( + $node_subscriber, + 'commit_count = 4', + 'the commit_count increment by commit prepared'); + +# STREAM COMMIT +$node_publisher->safe_psql( + 'postgres', qq[ +BEGIN; +INSERT INTO test_tab1 VALUES(generate_series(1001, 2000)); +COMMIT; +]); +confirm_transaction_stats_update( + $node_subscriber, + 'commit_count = 5', + 'the commit_count increment by stream commit'); + +# STREAM PREPARE & COMMIT PREPARED +$node_publisher->safe_psql( + 'postgres', qq[ +BEGIN; +INSERT INTO test_tab1 VALUES(generate_series(2001, 3000)); +PREPARE TRANSACTION 'gid2' +]); +$node_publisher->safe_psql('postgres', "COMMIT PREPARED 'gid2'"); +confirm_transaction_stats_update( + $node_subscriber, + 'commit_count = 6', + 'the commit_count increment by streamed commit prepared'); + +# ROLLBACK PREPARED +$node_publisher->safe_psql( + 'postgres', qq[ +BEGIN; +INSERT INTO test_tab1 VALUES (3); +PREPARE TRANSACTION 'gid3'; +]); +$node_publisher->safe_psql('postgres', "ROLLBACK PREPARED 'gid3'"); +confirm_transaction_stats_update( + $node_subscriber, + 'abort_count = 1', + 'the abort_count increment by rollback prepared'); + +# STREAM PREPARE & ROLLBACK PREPARED +$node_publisher->safe_psql( + 'postgres', qq[ +BEGIN; +INSERT INTO test_tab1 VALUES (generate_series(3001, 4000)); +PREPARE TRANSACTION 'gid4'; +]); +$node_publisher->safe_psql('postgres', "ROLLBACK PREPARED 'gid4'"); +confirm_transaction_stats_update( + $node_subscriber, + 'abort_count = 2', + 'the abort_count increment by streamed rollback prepared'); + +# There shouldn't be any records in the view after dropping the subscription. $node_subscriber->safe_psql('postgres', "DROP SUBSCRIPTION tap_sub;"); $result = $node_subscriber->safe_psql('postgres', "SELECT count(1) FROM pg_stat_subscription_workers"); -is($result, q(0), 'no error after dropping subscription'); +is($result, q(0), 'no record after dropping subscription'); $node_subscriber->stop('fast'); $node_publisher->stop('fast'); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f41ef0d..2d195ae 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -1945,6 +1945,7 @@ PgStat_MsgResetslrucounter PgStat_MsgSLRU PgStat_MsgSubscriptionPurge PgStat_MsgSubWorkerError +PgStat_MsgSubWorkerXactEnd PgStat_MsgTabpurge PgStat_MsgTabstat PgStat_MsgTempFile -- 1.8.3.1 ^ permalink raw reply [nested|flat] 113+ messages in thread
* Re: Failed transaction statistics to measure the logical replication progress @ 2021-12-13 09:19 Amit Kapila <[email protected]> parent: [email protected] <[email protected]> 1 sibling, 2 replies; 113+ messages in thread From: Amit Kapila @ 2021-12-13 09:19 UTC (permalink / raw) To: [email protected] <[email protected]>; +Cc: vignesh C <[email protected]>; Masahiko Sawada <[email protected]>; Greg Nancarrow <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]> On Tue, Dec 7, 2021 at 3:12 PM [email protected] <[email protected]> wrote: > Few questions and comments: ======================== 1. The <structname>pg_stat_subscription_workers</structname> view will contain one row per subscription worker on which errors have occurred, for workers applying logical replication changes and workers handling the initial data - copy of the subscribed tables. The statistics entry is removed when the - corresponding subscription is dropped. + copy of the subscribed tables. Also, the row corresponding to the apply + worker shows all transaction statistics of both types of workers on the + subscription. The statistics entry is removed when the corresponding + subscription is dropped. Why did you choose to show stats for both types of workers in one row? 2. + PGSTAT_MTYPE_SUBWORKERXACTEND, } StatMsgType; I don't think we comma with the last message type. 3. + Oid m_subrelid; + + /* necessary to determine column to increment */ + LogicalRepMsgType m_command; + +} PgStat_MsgSubWorkerXactEnd; Is m_subrelid used in this patch? If not, why did you keep it? I think if you choose to show separate stats for table sync and apply worker then probably it will be used. 4. /* + * Cumulative transaction statistics of subscription worker + */ + PgStat_Counter commit_count; + PgStat_Counter error_count; + PgStat_Counter abort_count; + I think it is better to keep the order of columns as commit_count, abort_count, error_count in the entire patch. -- With Regards, Amit Kapila. ^ permalink raw reply [nested|flat] 113+ messages in thread
* RE: Failed transaction statistics to measure the logical replication progress @ 2021-12-13 12:18 [email protected] <[email protected]> parent: Amit Kapila <[email protected]> 1 sibling, 1 reply; 113+ messages in thread From: [email protected] @ 2021-12-13 12:18 UTC (permalink / raw) To: 'Amit Kapila' <[email protected]>; +Cc: vignesh C <[email protected]>; Masahiko Sawada <[email protected]>; Greg Nancarrow <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]> On Monday, December 13, 2021 6:19 PM Amit Kapila <[email protected]> wrote: > On Tue, Dec 7, 2021 at 3:12 PM [email protected] > <[email protected]> wrote: > > Few questions and comments: Thank you for your comments ! > ======================== > 1. > The <structname>pg_stat_subscription_workers</structname> view will > contain > one row per subscription worker on which errors have occurred, for workers > applying logical replication changes and workers handling the initial data > - copy of the subscribed tables. The statistics entry is removed when the > - corresponding subscription is dropped. > + copy of the subscribed tables. Also, the row corresponding to the apply > + worker shows all transaction statistics of both types of workers on the > + subscription. The statistics entry is removed when the corresponding > + subscription is dropped. > > Why did you choose to show stats for both types of workers in one row? This is because if we have hundreds or thousands of tables for table sync, we need to create many entries to cover them and store the entries for all tables. > 2. > + PGSTAT_MTYPE_SUBWORKERXACTEND, > } StatMsgType; > > I don't think we comma with the last message type. > 4. > /* > + * Cumulative transaction statistics of subscription worker */ > + PgStat_Counter commit_count; PgStat_Counter error_count; > + PgStat_Counter abort_count; > + > > I think it is better to keep the order of columns as commit_count, abort_count, > error_count in the entire patch. Okay, I'll fix both points in the next version. > 3. > + Oid m_subrelid; > + > + /* necessary to determine column to increment */ LogicalRepMsgType > + m_command; > + > +} PgStat_MsgSubWorkerXactEnd; > > Is m_subrelid used in this patch? If not, why did you keep it? Absolutely, this was a mistake when I took the decision to merge both stats of table sync and apply worker. > I think if you choose > to show separate stats for table sync and apply worker then probably it will be > used. Yeah, I'll fix this. Of course, after I could confirm that the idea for merging the two types of workers stats was acceptable for you and others. Best Regards, Takamichi Osumi ^ permalink raw reply [nested|flat] 113+ messages in thread
* Re: Failed transaction statistics to measure the logical replication progress @ 2021-12-13 15:45 vignesh C <[email protected]> parent: [email protected] <[email protected]> 1 sibling, 1 reply; 113+ messages in thread From: vignesh C @ 2021-12-13 15:45 UTC (permalink / raw) To: [email protected] <[email protected]>; +Cc: Masahiko Sawada <[email protected]>; Amit Kapila <[email protected]>; Greg Nancarrow <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]> On Tue, Dec 7, 2021 at 3:12 PM [email protected] <[email protected]> wrote: > > On Monday, December 6, 2021 11:27 PM vignesh C <[email protected]> wrote: > > Thanks for the updated patch, few comments: > Thank you for your review ! > > > 1) We can keep the documentation similar to mention the count includes both > > table sync worker / main apply worker in case of commit_count/error_count > > and abort_count to keep it consistent. > > + <structfield>commit_count</structfield> <type>bigint</type> > > + </para> > > + <para> > > + Number of transactions successfully applied in this subscription. > > + COMMIT and COMMIT PREPARED increments this counter. > > + </para></entry> > > + </row> > > + > > + <row> > > + <entry role="catalog_table_entry"><para role="column_definition"> > > + <structfield>error_count</structfield> <type>bigint</type> > > + </para> > > + <para> > > + Number of transactions that failed to be applied by the table > > + sync worker or main apply worker in this subscription. > > + </para></entry> > > + </row> > > + > > + <row> > > + <entry role="catalog_table_entry"><para role="column_definition"> > > + <structfield>abort_count</structfield> <type>bigint</type> > > + </para> > > + <para> > > + Number of transactions aborted in this subscription. > > + ROLLBACK PREPARED increments this counter. > > + </para></entry> > > + </row> > Yeah, you are right. Fixed. > Note that abort_count is not used by table sync worker. > > > > 2) Can this be changed: > > + /* > > + * If this is a new error reported by table sync worker, > > consolidate this > > + * error count into the entry of apply worker. > > + */ > > + if (OidIsValid(msg->m_subrelid)) > > + { > > + /* Gain the apply worker stats */ > > + subwentry = pgstat_get_subworker_entry(dbentry, > > + msg->m_subid, > > + > > InvalidOid, true); > > + subwentry->error_count++; > > + } > > + else > > + subwentry->error_count++; /* increment the apply > > worker's counter. */ > > To: > > + /* > > + * If this is a new error reported by table sync worker, > > consolidate this > > + * error count into the entry of apply worker. > > + */ > > + if (OidIsValid(msg->m_subrelid)) > > + /* Gain the apply worker stats */ > > + subwentry = pgstat_get_subworker_entry(dbentry, > > + msg->m_subid, > > + > > InvalidOid, true); > > + > > + subwentry->error_count++; /* increment the apply > > worker's counter. */ > Your suggestion looks better. > Also, I fixed some comments of this part > so that we don't need to add a separate comment at the bottom > for the increment of the apply worker. > > > > 3) Since both 026_worker_stats and 027_worker_xact_stats.pl are testing > > pg_stat_subscription_workers, can we move the tests to 026_worker_stats.pl. > > If possible the error_count validation can be combined with the existing tests. > > diff --git a/src/test/subscription/t/027_worker_xact_stats.pl > > b/src/test/subscription/t/027_worker_xact_stats.pl > > new file mode 100644 > > index 0000000..31dbea1 > > --- /dev/null > > +++ b/src/test/subscription/t/027_worker_xact_stats.pl > > @@ -0,0 +1,162 @@ > > + > > +# Copyright (c) 2021, PostgreSQL Global Development Group > > + > > +# Tests for subscription worker statistics during apply. > > +use strict; > > +use warnings; > > +use PostgreSQL::Test::Cluster; > > +use PostgreSQL::Test::Utils; > > +use Test::More tests => 1; > > + > > +# Create publisher node > Right. I've integrated my tests with 026_worker_stats.pl. > I think error_count validations are combined as you suggested. > Another change I did is to introduce one function > to contribute to better readability of the stats tests. > > Here, the 026_worker_stats.pl didn't look aligned by > pgperltidy. This is not a serious issue at all. > Yet, when I ran pgperltidy, the existing codes > that required adjustments came into my patch. > Therefore, I made a separate part for this. Thanks for the updated patch, few comments: 1) Can we change this: /* + * Report the success of table sync as one commit to consolidate all + * transaction stats into one record. + */ + pgstat_report_subworker_xact_end(MyLogicalRepWorker->subid, + LOGICAL_REP_MSG_COMMIT); + To: /* Report the success of table sync */ + pgstat_report_subworker_xact_end(MyLogicalRepWorker->subid, + LOGICAL_REP_MSG_COMMIT); + 2) Typo: ealier should be earlier + /* + * Report ealier than the call of process_syncing_tables() not to miss an + * increment of commit_count in case it leads to the process exit. See + * process_syncing_tables_for_apply(). + */ + pgstat_report_subworker_xact_end(MyLogicalRepWorker->subid, + LOGICAL_REP_MSG_COMMIT); + 3) Should we add an Assert for subwentry: + /* + * If this is a new error reported by table sync worker, consolidate this + * error count into the entry of apply worker, by swapping the stats + * entries. + */ + if (OidIsValid(msg->m_subrelid)) + subwentry = pgstat_get_subworker_entry(dbentry, msg->m_subid, + InvalidOid, true); + subwentry->error_count++; 4) Can we slightly change it to :We can change it: +# Check the update of stats counters. +confirm_transaction_stats_update( + $node_subscriber, + 'commit_count = 1', + 'the commit_count increment by table sync'); + +confirm_transaction_stats_update( + $node_subscriber, + 'error_count = 1', + 'the error_count increment by table sync'); to: +# Check updation of subscription worker transaction count statistics. +confirm_transaction_stats_update( + $node_subscriber, + 'commit_count = 1', + 'check table sync worker commit count is updated'); + +confirm_transaction_stats_update( + $node_subscriber, + 'error_count = 1', + 'check table sync worker error count is updated'); Regards, Vignesh ^ permalink raw reply [nested|flat] 113+ messages in thread
* Re: Failed transaction statistics to measure the logical replication progress @ 2021-12-14 02:28 Amit Kapila <[email protected]> parent: [email protected] <[email protected]> 0 siblings, 4 replies; 113+ messages in thread From: Amit Kapila @ 2021-12-14 02:28 UTC (permalink / raw) To: [email protected] <[email protected]>; +Cc: vignesh C <[email protected]>; Masahiko Sawada <[email protected]>; Greg Nancarrow <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]> On Mon, Dec 13, 2021 at 5:48 PM [email protected] <[email protected]> wrote: > > On Monday, December 13, 2021 6:19 PM Amit Kapila <[email protected]> wrote: > > On Tue, Dec 7, 2021 at 3:12 PM [email protected] > > <[email protected]> wrote: > > > > Few questions and comments: > Thank you for your comments ! > > > ======================== > > 1. > > The <structname>pg_stat_subscription_workers</structname> view will > > contain > > one row per subscription worker on which errors have occurred, for workers > > applying logical replication changes and workers handling the initial data > > - copy of the subscribed tables. The statistics entry is removed when the > > - corresponding subscription is dropped. > > + copy of the subscribed tables. Also, the row corresponding to the apply > > + worker shows all transaction statistics of both types of workers on the > > + subscription. The statistics entry is removed when the corresponding > > + subscription is dropped. > > > > Why did you choose to show stats for both types of workers in one row? > This is because if we have hundreds or thousands of tables for table sync, > we need to create many entries to cover them and store the entries for all tables. > If we fear a large number of entries for such workers then won't it be better to show the value of these stats only for apply workers. I think normally the table sync workers perform only copy operation or maybe a fixed number of xacts, so, one might not be interested in the transaction stats of these workers. I find merging only specific stats of two different types of workers confusing. What do others think about this? -- With Regards, Amit Kapila. ^ permalink raw reply [nested|flat] 113+ messages in thread
* RE: Failed transaction statistics to measure the logical replication progress @ 2021-12-14 13:28 [email protected] <[email protected]> parent: Amit Kapila <[email protected]> 3 siblings, 0 replies; 113+ messages in thread From: [email protected] @ 2021-12-14 13:28 UTC (permalink / raw) To: Amit Kapila <[email protected]>; +Cc: vignesh C <[email protected]>; Masahiko Sawada <[email protected]>; Greg Nancarrow <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]> On Tues, Dec 14, 2021 10:28 AM Amit Kapila <[email protected]> wrote: > On Mon, Dec 13, 2021 at 5:48 PM [email protected] > <[email protected]> wrote: > > > > On Monday, December 13, 2021 6:19 PM Amit Kapila > <[email protected]> wrote: > > > On Tue, Dec 7, 2021 at 3:12 PM [email protected] > > > <[email protected]> wrote: > > > > > > Few questions and comments: > > Thank you for your comments ! > > > > > ======================== > > > 1. > > > one row per subscription worker on which errors have occurred, for workers > > > applying logical replication changes and workers handling the initial data > > > - copy of the subscribed tables. The statistics entry is removed when the > > > - corresponding subscription is dropped. > > > + copy of the subscribed tables. Also, the row corresponding to the apply > > > + worker shows all transaction statistics of both types of workers on the > > > + subscription. The statistics entry is removed when the corresponding > > > + subscription is dropped. > > > > > > Why did you choose to show stats for both types of workers in one row? > > This is because if we have hundreds or thousands of tables for table sync, > > we need to create many entries to cover them and store the entries for all > > tables. > > > > If we fear a large number of entries for such workers then won't it be > better to show the value of these stats for apply workers. I > think normally the table sync workers perform only copy operation or > maybe a fixed number of xacts, so, one might not be interested in the > transaction stats of these workers. I find merging only specific stats > of two different types of workers confusing. > > What do others think about this? Personally, I agreed that merging two types of stats into one row might not be a good idea. And the xact stats of table sync workers are usually less interesting than the apply worker's, So, it's seems acceptable to me if we show stats only for apply workers and document about this. Best regards, Hou zj ^ permalink raw reply [nested|flat] 113+ messages in thread
* Re: Failed transaction statistics to measure the logical replication progress @ 2021-12-15 05:09 Masahiko Sawada <[email protected]> parent: Amit Kapila <[email protected]> 3 siblings, 0 replies; 113+ messages in thread From: Masahiko Sawada @ 2021-12-15 05:09 UTC (permalink / raw) To: Amit Kapila <[email protected]>; +Cc: [email protected] <[email protected]>; vignesh C <[email protected]>; Greg Nancarrow <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]> On Tue, Dec 14, 2021 at 11:28 AM Amit Kapila <[email protected]> wrote: > > On Mon, Dec 13, 2021 at 5:48 PM [email protected] > <[email protected]> wrote: > > > > On Monday, December 13, 2021 6:19 PM Amit Kapila <[email protected]> wrote: > > > On Tue, Dec 7, 2021 at 3:12 PM [email protected] > > > <[email protected]> wrote: > > > > > > Few questions and comments: > > Thank you for your comments ! > > > > > ======================== > > > 1. > > > The <structname>pg_stat_subscription_workers</structname> view will > > > contain > > > one row per subscription worker on which errors have occurred, for workers > > > applying logical replication changes and workers handling the initial data > > > - copy of the subscribed tables. The statistics entry is removed when the > > > - corresponding subscription is dropped. > > > + copy of the subscribed tables. Also, the row corresponding to the apply > > > + worker shows all transaction statistics of both types of workers on the > > > + subscription. The statistics entry is removed when the corresponding > > > + subscription is dropped. > > > > > > Why did you choose to show stats for both types of workers in one row? > > This is because if we have hundreds or thousands of tables for table sync, > > we need to create many entries to cover them and store the entries for all tables. > > > > If we fear a large number of entries for such workers then won't it be > better to show the value of these stats only for apply workers. I > think normally the table sync workers perform only copy operation or > maybe a fixed number of xacts, so, one might not be interested in the > transaction stats of these workers. I find merging only specific stats > of two different types of workers confusing. > > What do others think about this? I understand the concern to have a large number of entries but I agree that merging only specific stats would confuse users. As Amit suggested, it'd be better to show only apply workers' transaction stats. Regards, -- Masahiko Sawada EDB: https://www.enterprisedb.com/ ^ permalink raw reply [nested|flat] 113+ messages in thread
* Re: Failed transaction statistics to measure the logical replication progress @ 2021-12-15 12:51 vignesh C <[email protected]> parent: Amit Kapila <[email protected]> 3 siblings, 1 reply; 113+ messages in thread From: vignesh C @ 2021-12-15 12:51 UTC (permalink / raw) To: Amit Kapila <[email protected]>; +Cc: [email protected] <[email protected]>; Masahiko Sawada <[email protected]>; Greg Nancarrow <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]> On Tue, Dec 14, 2021 at 7:58 AM Amit Kapila <[email protected]> wrote: > > On Mon, Dec 13, 2021 at 5:48 PM [email protected] > <[email protected]> wrote: > > > > On Monday, December 13, 2021 6:19 PM Amit Kapila <[email protected]> wrote: > > > On Tue, Dec 7, 2021 at 3:12 PM [email protected] > > > <[email protected]> wrote: > > > > > > Few questions and comments: > > Thank you for your comments ! > > > > > ======================== > > > 1. > > > The <structname>pg_stat_subscription_workers</structname> view will > > > contain > > > one row per subscription worker on which errors have occurred, for workers > > > applying logical replication changes and workers handling the initial data > > > - copy of the subscribed tables. The statistics entry is removed when the > > > - corresponding subscription is dropped. > > > + copy of the subscribed tables. Also, the row corresponding to the apply > > > + worker shows all transaction statistics of both types of workers on the > > > + subscription. The statistics entry is removed when the corresponding > > > + subscription is dropped. > > > > > > Why did you choose to show stats for both types of workers in one row? > > This is because if we have hundreds or thousands of tables for table sync, > > we need to create many entries to cover them and store the entries for all tables. > > > > If we fear a large number of entries for such workers then won't it be > better to show the value of these stats only for apply workers. I > think normally the table sync workers perform only copy operation or > maybe a fixed number of xacts, so, one might not be interested in the > transaction stats of these workers. I find merging only specific stats > of two different types of workers confusing. > > What do others think about this? We can remove the table sync workers transaction stats count to avoid confusion, take care of the documentation changes too accordingly. Regards, Vignesh ^ permalink raw reply [nested|flat] 113+ messages in thread
* RE: Failed transaction statistics to measure the logical replication progress @ 2021-12-16 06:59 [email protected] <[email protected]> parent: vignesh C <[email protected]> 0 siblings, 1 reply; 113+ messages in thread From: [email protected] @ 2021-12-16 06:59 UTC (permalink / raw) To: 'vignesh C' <[email protected]>; Amit Kapila <[email protected]>; +Cc: Masahiko Sawada <[email protected]>; Greg Nancarrow <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]> On Wednesday, December 15, 2021 9:52 PM vignesh C <[email protected]> wrote: > On Tue, Dec 14, 2021 at 7:58 AM Amit Kapila <[email protected]> > wrote: > > > > On Mon, Dec 13, 2021 at 5:48 PM [email protected] > > <[email protected]> wrote: > > > > > > On Monday, December 13, 2021 6:19 PM Amit Kapila > <[email protected]> wrote: > > > > On Tue, Dec 7, 2021 at 3:12 PM [email protected] > > > > <[email protected]> wrote: > > > > > > > > Few questions and comments: > > > Thank you for your comments ! > > > > > > > ======================== > > > > 1. > > > > The <structname>pg_stat_subscription_workers</structname> view > > > > will contain > > > > one row per subscription worker on which errors have occurred, for > workers > > > > applying logical replication changes and workers handling the initial > data > > > > - copy of the subscribed tables. The statistics entry is removed > when the > > > > - corresponding subscription is dropped. > > > > + copy of the subscribed tables. Also, the row corresponding to the > apply > > > > + worker shows all transaction statistics of both types of workers on > the > > > > + subscription. The statistics entry is removed when the > corresponding > > > > + subscription is dropped. > > > > > > > > Why did you choose to show stats for both types of workers in one row? > > > This is because if we have hundreds or thousands of tables for table > > > sync, we need to create many entries to cover them and store the entries for > all tables. > > > > > > > If we fear a large number of entries for such workers then won't it be > > better to show the value of these stats only for apply workers. I > > think normally the table sync workers perform only copy operation or > > maybe a fixed number of xacts, so, one might not be interested in the > > transaction stats of these workers. I find merging only specific stats > > of two different types of workers confusing. > > > > What do others think about this? > > We can remove the table sync workers transaction stats count to avoid > confusion, take care of the documentation changes too accordingly. Hi, apologies for my late reply. Thank you, everyone for confirming the direction. I'll follow the consensus of the community and fix the patch, including other comments. I'll treat only the stats for apply workers. Best Regards, Takamichi Osumi ^ permalink raw reply [nested|flat] 113+ messages in thread
* Re: Failed transaction statistics to measure the logical replication progress @ 2021-12-16 07:08 Greg Nancarrow <[email protected]> parent: Amit Kapila <[email protected]> 3 siblings, 0 replies; 113+ messages in thread From: Greg Nancarrow @ 2021-12-16 07:08 UTC (permalink / raw) To: Amit Kapila <[email protected]>; +Cc: [email protected] <[email protected]>; vignesh C <[email protected]>; Masahiko Sawada <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]> On Tue, Dec 14, 2021 at 1:28 PM Amit Kapila <[email protected]> wrote: > > If we fear a large number of entries for such workers then won't it be > better to show the value of these stats only for apply workers. I > think normally the table sync workers perform only copy operation or > maybe a fixed number of xacts, so, one might not be interested in the > transaction stats of these workers. I find merging only specific stats > of two different types of workers confusing. > > What do others think about this? > I think it might be OK to NOT include the transaction stats of the tablesync workers, but my understanding (and slight concern) is that currently there is potentially some overlap in the work done by the tablesync and apply workers - perhaps the small patch (see [1]) proposed by Peter Smith could also be considered, in order to make that distinction of work clearer, and the stats more meaningful? ---- [1] https://www.postgresql.org/message-id/flat/CAHut+Pt39PbQs0SxT9RMM89aYiZoQ0Kw46YZSkKZwK8z5HOr3g@mail.... Regards, Greg Nancarrow Fujitsu Australia ^ permalink raw reply [nested|flat] 113+ messages in thread
* RE: Failed transaction statistics to measure the logical replication progress @ 2021-12-16 11:36 [email protected] <[email protected]> parent: [email protected] <[email protected]> 0 siblings, 2 replies; 113+ messages in thread From: [email protected] @ 2021-12-16 11:36 UTC (permalink / raw) To: 'vignesh C' <[email protected]>; Amit Kapila <[email protected]>; +Cc: Masahiko Sawada <[email protected]>; Greg Nancarrow <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]> On Thursday, December 16, 2021 4:00 PM I wrote: > Thank you, everyone for confirming the direction. > I'll follow the consensus of the community and fix the patch, including other > comments. > I'll treat only the stats for apply workers. Hi, created a new version v17 according to the recent discussion with changes to address other review comments. Kindly have a look at it. Best Regards, Takamichi Osumi Attachments: [application/octet-stream] v17-0002-Extend-pg_stat_subscription_workers-to-include-g.patch (23.1K, ../../TYCPR01MB83734A7A0596AC7ADB0DCB51ED779@TYCPR01MB8373.jpnprd01.prod.outlook.com/2-v17-0002-Extend-pg_stat_subscription_workers-to-include-g.patch) download | inline diff: From a28de2f6a92cf4b72246edbc57484873652ae286 Mon Sep 17 00:00:00 2001 From: Takamichi Osumi <[email protected]> Date: Thu, 16 Dec 2021 11:11:55 +0000 Subject: [PATCH v17 2/2] Extend pg_stat_subscription_workers to include general transaction statistics Categorize transactions of logical replication subscriber into three types (commit, abort, error) and introduce cumulative columns of those numbers in the pg_stat_subscription_workers. In order to avoid having a large number of entries to be created by the table synchronization, the new stats columns are utilized only by the apply worker. Author: Takamichi Osumi Reviewed-by: Amit Kapila, Masahiko Sawada, Hou Zhijie, Greg Nancarrow, Vignesh C, Ajin Cherian Discussion: https://www.postgresql.org/message-id/OSBPR01MB48887CA8F40C8D984A6DC00CED199%40OSBPR01MB4888.jpnprd01.prod.outlook.com --- doc/src/sgml/monitoring.sgml | 42 ++++++++++- src/backend/catalog/system_views.sql | 3 + src/backend/postmaster/pgstat.c | 74 +++++++++++++++++++ src/backend/replication/logical/worker.c | 16 ++++ src/backend/utils/adt/pgstatfuncs.c | 25 +++++-- src/include/catalog/pg_proc.dat | 6 +- src/include/pgstat.h | 28 +++++++ src/test/regress/expected/rules.out | 5 +- src/test/subscription/t/026_worker_stats.pl | 109 +++++++++++++++++++++++++++- src/tools/pgindent/typedefs.list | 1 + 10 files changed, 290 insertions(+), 19 deletions(-) diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml index 62f2a33..e795a1d 100644 --- a/doc/src/sgml/monitoring.sgml +++ b/doc/src/sgml/monitoring.sgml @@ -629,8 +629,8 @@ postgres 27093 0.0 0.0 30096 2752 ? Ss 11:34 0:00 postgres: ser <row> <entry><structname>pg_stat_subscription_workers</structname><indexterm><primary>pg_stat_subscription_workers</primary></indexterm></entry> - <entry>One row per subscription worker, showing statistics about errors - that occurred on that subscription worker. + <entry>One row per subscription worker, showing statistics about transactions + and errors that occurred on that subscription worker. See <link linkend="monitoring-pg-stat-subscription-workers"> <structname>pg_stat_subscription_workers</structname></link> for details. </entry> @@ -3074,8 +3074,10 @@ SELECT pid, wait_event_type, wait_event FROM pg_stat_activity WHERE wait_event i The <structname>pg_stat_subscription_workers</structname> view will contain one row per subscription worker on which errors have occurred, for workers applying logical replication changes and workers handling the initial data - copy of the subscribed tables. The statistics entry is removed when the - corresponding subscription is dropped. + copy of the subscribed tables. The row corresponding to the apply + worker shows transaction statistics of the main apply worker on the + subscription. The statistics entry is removed when the corresponding + subscription is dropped. </para> <table id="pg-stat-subscription-workers" xreflabel="pg_stat_subscription_workers"> @@ -3123,6 +3125,38 @@ SELECT pid, wait_event_type, wait_event FROM pg_stat_activity WHERE wait_event i <row> <entry role="catalog_table_entry"><para role="column_definition"> + <structfield>commit_count</structfield> <type>bigint</type> + </para> + <para> + Number of transactions successfully applied in this subscription. + Both COMMIT and COMMIT PREPARED increment this counter. + </para></entry> + </row> + + <row> + <entry role="catalog_table_entry"><para role="column_definition"> + <structfield>abort_count</structfield> <type>bigint</type> + </para> + <para> + Number of transactions aborted in this subscription. + ROLLBACK PREPARED increments this counter. + </para></entry> + </row> + + <row> + <entry role="catalog_table_entry"><para role="column_definition"> + <structfield>error_count</structfield> <type>bigint</type> + </para> + <para> + Number of transactions that failed to be applied by the table + sync worker or main apply worker in this subscription. This + counter is updated after confirming the error is not same as + the previous one. + </para></entry> + </row> + + <row> + <entry role="catalog_table_entry"><para role="column_definition"> <structfield>last_error_relid</structfield> <type>oid</type> </para> <para> diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index 61b515c..dda3cd7 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1267,6 +1267,9 @@ CREATE VIEW pg_stat_subscription_workers AS w.subid, s.subname, w.subrelid, + w.commit_count, + w.abort_count, + w.error_count, w.last_error_relid, w.last_error_command, w.last_error_xid, diff --git a/src/backend/postmaster/pgstat.c b/src/backend/postmaster/pgstat.c index 7264d2c..bac41ca 100644 --- a/src/backend/postmaster/pgstat.c +++ b/src/backend/postmaster/pgstat.c @@ -382,6 +382,7 @@ static void pgstat_recv_replslot(PgStat_MsgReplSlot *msg, int len); static void pgstat_recv_tempfile(PgStat_MsgTempFile *msg, int len); static void pgstat_recv_subscription_purge(PgStat_MsgSubscriptionPurge *msg, int len); static void pgstat_recv_subworker_error(PgStat_MsgSubWorkerError *msg, int len); +static void pgstat_recv_subworker_xact_end(PgStat_MsgSubWorkerXactEnd *msg, int len); /* ------------------------------------------------------------ * Public functions called from postmaster follow @@ -1949,6 +1950,32 @@ pgstat_report_replslot_drop(const char *slotname) } /* ---------- + * pgstat_report_subworker_xact_end() - + * + * Tell the collector that worker transaction has successfully completed. + * This should be called before the call of process_syning_tables() not to + * miss an increment of transaction stats in case it leads to a process exit. + * See process_syncing_tables_for_apply(). + * ---------- + */ +void +pgstat_report_subworker_xact_end(Oid subid, LogicalRepMsgType command) +{ + PgStat_MsgSubWorkerXactEnd msg; + + Assert(command == LOGICAL_REP_MSG_COMMIT || + command == LOGICAL_REP_MSG_STREAM_COMMIT || + command == LOGICAL_REP_MSG_COMMIT_PREPARED || + command == LOGICAL_REP_MSG_ROLLBACK_PREPARED); + + pgstat_setheader(&msg.m_hdr, PGSTAT_MTYPE_SUBWORKERXACTEND); + msg.m_databaseid = MyDatabaseId; + msg.m_subid = subid; + msg.m_command = command; + pgstat_send(&msg, sizeof(PgStat_MsgSubWorkerXactEnd)); +} + +/* ---------- * pgstat_report_subworker_error() - * * Tell the collector about the subscription worker error. @@ -3746,6 +3773,10 @@ PgstatCollectorMain(int argc, char *argv[]) pgstat_recv_subworker_error(&msg.msg_subworkererror, len); break; + case PGSTAT_MTYPE_SUBWORKERXACTEND: + pgstat_recv_subworker_xact_end(&msg.msg_subworkerxactend, len); + break; + default: break; } @@ -3965,6 +3996,9 @@ pgstat_get_subworker_entry(PgStat_StatDBEntry *dbentry, Oid subid, Oid subrelid, /* If not found, initialize the new one */ if (!found) { + subwentry->commit_count = 0; + subwentry->abort_count = 0; + subwentry->error_count = 0; subwentry->last_error_relid = InvalidOid; subwentry->last_error_command = 0; subwentry->last_error_xid = InvalidTransactionId; @@ -6153,6 +6187,39 @@ pgstat_recv_subscription_purge(PgStat_MsgSubscriptionPurge *msg, int len) } /* ---------- + * pgstat_recv_subworker_xact_end() - + * + * Process a SUBWORKERXACTEND message. + * ---------- + */ +static void +pgstat_recv_subworker_xact_end(PgStat_MsgSubWorkerXactEnd *msg, int len) +{ + PgStat_StatDBEntry *dbentry; + PgStat_StatSubWorkerEntry *wentry; + + dbentry = pgstat_get_db_entry(msg->m_databaseid, true); + wentry = pgstat_get_subworker_entry(dbentry, msg->m_subid, + InvalidOid, true); + Assert(wentry); + + switch (msg->m_command) + { + case LOGICAL_REP_MSG_COMMIT: + case LOGICAL_REP_MSG_STREAM_COMMIT: + case LOGICAL_REP_MSG_COMMIT_PREPARED: + wentry->commit_count++; + break; + case LOGICAL_REP_MSG_ROLLBACK_PREPARED: + wentry->abort_count++; + break; + default: + elog(ERROR, "unexpected logical message type as transaction end"); + break; + } +} + +/* ---------- * pgstat_recv_subworker_error() - * * Process a SUBWORKERERROR message. @@ -6193,6 +6260,13 @@ pgstat_recv_subworker_error(PgStat_MsgSubWorkerError *msg, int len) subwentry->last_error_time = msg->m_timestamp; strlcpy(subwentry->last_error_message, msg->m_message, PGSTAT_SUBWORKERERROR_MSGLEN); + + /* + * Only if this is a new error reported by the apply worker, increment the + * counter of error. + */ + if (!OidIsValid(msg->m_subrelid)) + subwentry->error_count++; } /* ---------- diff --git a/src/backend/replication/logical/worker.c b/src/backend/replication/logical/worker.c index 2e79302..45e7bf4 100644 --- a/src/backend/replication/logical/worker.c +++ b/src/backend/replication/logical/worker.c @@ -815,6 +815,10 @@ apply_handle_commit(StringInfo s) apply_handle_commit_internal(&commit_data); + /* Update stats */ + pgstat_report_subworker_xact_end(MyLogicalRepWorker->subid, + LOGICAL_REP_MSG_COMMIT); + /* Process any tables that are being synchronized in parallel. */ process_syncing_tables(commit_data.end_lsn); @@ -960,6 +964,10 @@ apply_handle_commit_prepared(StringInfo s) store_flush_position(prepare_data.end_lsn); in_remote_transaction = false; + /* Update stats */ + pgstat_report_subworker_xact_end(MyLogicalRepWorker->subid, + LOGICAL_REP_MSG_COMMIT_PREPARED); + /* Process any tables that are being synchronized in parallel. */ process_syncing_tables(prepare_data.end_lsn); @@ -1011,6 +1019,10 @@ apply_handle_rollback_prepared(StringInfo s) store_flush_position(rollback_data.rollback_end_lsn); in_remote_transaction = false; + /* Update stats */ + pgstat_report_subworker_xact_end(MyLogicalRepWorker->subid, + LOGICAL_REP_MSG_ROLLBACK_PREPARED); + /* Process any tables that are being synchronized in parallel. */ process_syncing_tables(rollback_data.rollback_end_lsn); @@ -1435,6 +1447,10 @@ apply_handle_stream_commit(StringInfo s) /* unlink the files with serialized changes and subxact info */ stream_cleanup_files(MyLogicalRepWorker->subid, xid); + /* Update stats */ + pgstat_report_subworker_xact_end(MyLogicalRepWorker->subid, + LOGICAL_REP_MSG_STREAM_COMMIT); + /* Process any tables that are being synchronized in parallel. */ process_syncing_tables(commit_data.end_lsn); diff --git a/src/backend/utils/adt/pgstatfuncs.c b/src/backend/utils/adt/pgstatfuncs.c index f529c15..87b5254 100644 --- a/src/backend/utils/adt/pgstatfuncs.c +++ b/src/backend/utils/adt/pgstatfuncs.c @@ -2415,7 +2415,7 @@ pg_stat_get_replication_slot(PG_FUNCTION_ARGS) Datum pg_stat_get_subscription_worker(PG_FUNCTION_ARGS) { -#define PG_STAT_GET_SUBSCRIPTION_WORKER_COLS 8 +#define PG_STAT_GET_SUBSCRIPTION_WORKER_COLS 11 Oid subid = PG_GETARG_OID(0); Oid subrelid; TupleDesc tupdesc; @@ -2442,17 +2442,23 @@ pg_stat_get_subscription_worker(PG_FUNCTION_ARGS) OIDOID, -1, 0); TupleDescInitEntry(tupdesc, (AttrNumber) 2, "subrelid", OIDOID, -1, 0); - TupleDescInitEntry(tupdesc, (AttrNumber) 3, "last_error_relid", + TupleDescInitEntry(tupdesc, (AttrNumber) 3, "commit_count", + INT8OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) 4, "abort_count", + INT8OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) 5, "error_count", + INT8OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) 6, "last_error_relid", OIDOID, -1, 0); - TupleDescInitEntry(tupdesc, (AttrNumber) 4, "last_error_command", + TupleDescInitEntry(tupdesc, (AttrNumber) 7, "last_error_command", TEXTOID, -1, 0); - TupleDescInitEntry(tupdesc, (AttrNumber) 5, "last_error_xid", + TupleDescInitEntry(tupdesc, (AttrNumber) 8, "last_error_xid", XIDOID, -1, 0); - TupleDescInitEntry(tupdesc, (AttrNumber) 6, "last_error_count", + TupleDescInitEntry(tupdesc, (AttrNumber) 9, "last_error_count", INT8OID, -1, 0); - TupleDescInitEntry(tupdesc, (AttrNumber) 7, "last_error_message", + TupleDescInitEntry(tupdesc, (AttrNumber) 10, "last_error_message", TEXTOID, -1, 0); - TupleDescInitEntry(tupdesc, (AttrNumber) 8, "last_error_time", + TupleDescInitEntry(tupdesc, (AttrNumber) 11, "last_error_time", TIMESTAMPTZOID, -1, 0); BlessTupleDesc(tupdesc); @@ -2470,6 +2476,11 @@ pg_stat_get_subscription_worker(PG_FUNCTION_ARGS) else nulls[i++] = true; + /* transaction stats */ + values[i++] = Int64GetDatum(wentry->commit_count); + values[i++] = Int64GetDatum(wentry->abort_count); + values[i++] = Int64GetDatum(wentry->error_count); + /* last_error_relid */ if (OidIsValid(wentry->last_error_relid)) values[i++] = ObjectIdGetDatum(wentry->last_error_relid); diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index 4d992dc..b76f84a 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -5375,9 +5375,9 @@ proname => 'pg_stat_get_subscription_worker', prorows => '1', proisstrict => 'f', proretset => 't', provolatile => 's', proparallel => 'r', prorettype => 'record', proargtypes => 'oid oid', - proallargtypes => '{oid,oid,oid,oid,oid,text,xid,int8,text,timestamptz}', - proargmodes => '{i,i,o,o,o,o,o,o,o,o}', - proargnames => '{subid,subrelid,subid,subrelid,last_error_relid,last_error_command,last_error_xid,last_error_count,last_error_message,last_error_time}', + proallargtypes => '{oid,oid,oid,oid,int8,int8,int8,oid,text,xid,int8,text,timestamptz}', + proargmodes => '{i,i,o,o,o,o,o,o,o,o,o,o,o}', + proargnames => '{subid,subrelid,subid,subrelid,commit_count,abort_count,error_count,last_error_relid,last_error_command,last_error_xid,last_error_count,last_error_message,last_error_time}', prosrc => 'pg_stat_get_subscription_worker' }, { oid => '6118', descr => 'statistics: information about subscription', proname => 'pg_stat_get_subscription', prorows => '10', proisstrict => 'f', diff --git a/src/include/pgstat.h b/src/include/pgstat.h index 5b51b58..dd698ab 100644 --- a/src/include/pgstat.h +++ b/src/include/pgstat.h @@ -86,6 +86,7 @@ typedef enum StatMsgType PGSTAT_MTYPE_DISCONNECT, PGSTAT_MTYPE_SUBSCRIPTIONPURGE, PGSTAT_MTYPE_SUBWORKERERROR, + PGSTAT_MTYPE_SUBWORKERXACTEND } StatMsgType; /* ---------- @@ -558,6 +559,24 @@ typedef struct PgStat_MsgSubscriptionPurge } PgStat_MsgSubscriptionPurge; /* ---------- + * PgStat_MsgSubscriptionXactEnd Sent by the apply worker to report transaction + * ends. + * ---------- + */ +typedef struct PgStat_MsgSubWorkerXactEnd +{ + PgStat_MsgHdr m_hdr; + + /* determine the worker entry */ + Oid m_databaseid; + Oid m_subid; + + /* necessary to determine column to increment */ + LogicalRepMsgType m_command; + +} PgStat_MsgSubWorkerXactEnd; + +/* ---------- * PgStat_MsgSubWorkerError Sent by the apply worker or the table sync * worker to report the error occurred while * processing changes. @@ -769,6 +788,7 @@ typedef union PgStat_Msg PgStat_MsgDisconnect msg_disconnect; PgStat_MsgSubscriptionPurge msg_subscriptionpurge; PgStat_MsgSubWorkerError msg_subworkererror; + PgStat_MsgSubWorkerXactEnd msg_subworkerxactend; } PgStat_Msg; @@ -1010,6 +1030,13 @@ typedef struct PgStat_StatSubWorkerEntry PgStat_StatSubWorkerKey key; /* hash key (must be first) */ /* + * Cumulative transaction statistics of subscription worker + */ + PgStat_Counter commit_count; + PgStat_Counter abort_count; + PgStat_Counter error_count; + + /* * Subscription worker error statistics representing an error that * occurred during application of changes or the initial table * synchronization. @@ -1131,6 +1158,7 @@ extern void pgstat_report_checksum_failure(void); extern void pgstat_report_replslot(const PgStat_StatReplSlotEntry *repSlotStat); extern void pgstat_report_replslot_create(const char *slotname); extern void pgstat_report_replslot_drop(const char *slotname); +extern void pgstat_report_subworker_xact_end(Oid subid, LogicalRepMsgType command); extern void pgstat_report_subworker_error(Oid subid, Oid subrelid, Oid relid, LogicalRepMsgType command, TransactionId xid, const char *errmsg); diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out index b58b062..2b0bfae 100644 --- a/src/test/regress/expected/rules.out +++ b/src/test/regress/expected/rules.out @@ -2097,6 +2097,9 @@ pg_stat_subscription| SELECT su.oid AS subid, pg_stat_subscription_workers| SELECT w.subid, s.subname, w.subrelid, + w.commit_count, + w.abort_count, + w.error_count, w.last_error_relid, w.last_error_command, w.last_error_xid, @@ -2110,7 +2113,7 @@ pg_stat_subscription_workers| SELECT w.subid, SELECT pg_subscription_rel.srsubid AS subid, pg_subscription_rel.srrelid AS relid FROM pg_subscription_rel) sr, - (LATERAL pg_stat_get_subscription_worker(sr.subid, sr.relid) w(subid, subrelid, last_error_relid, last_error_command, last_error_xid, last_error_count, last_error_message, last_error_time) + (LATERAL pg_stat_get_subscription_worker(sr.subid, sr.relid) w(subid, subrelid, commit_count, abort_count, error_count, last_error_relid, last_error_command, last_error_xid, last_error_count, last_error_message, last_error_time) JOIN pg_subscription s ON ((w.subid = s.oid))); pg_stat_sys_indexes| SELECT pg_stat_all_indexes.relid, pg_stat_all_indexes.indexrelid, diff --git a/src/test/subscription/t/026_worker_stats.pl b/src/test/subscription/t/026_worker_stats.pl index 8005c54..99253f4 100644 --- a/src/test/subscription/t/026_worker_stats.pl +++ b/src/test/subscription/t/026_worker_stats.pl @@ -1,7 +1,7 @@ # Copyright (c) 2021, PostgreSQL Global Development Group -# Tests for subscription error stats. +# Tests for subscription stats. use strict; use warnings; use PostgreSQL::Test::Cluster; @@ -44,9 +44,32 @@ WHERE last_error_relid = '$relname'::regclass or die "Timed out while waiting for " . $msg; } +# Test whether the update of general transaction stats satisfies the expected +# condition or not. +sub confirm_transaction_stats_update +{ + my ($node, $condition, $msg) = @_; + + # Check only the stats of the apply worker + my $sql = qq[ +SELECT count(1) = 1 +FROM pg_stat_subscription_workers +WHERE subrelid IS NULL AND $condition]; + + $node->poll_query_until('postgres', $sql) + or die "Timed out while waiting for " . $msg; +} + # Create publisher node. my $node_publisher = PostgreSQL::Test::Cluster->new('publisher'); $node_publisher->init(allows_streaming => 'logical'); +$node_publisher->append_conf( + 'postgresql.conf', qq[ +logical_decoding_work_mem = 64kB +max_prepared_transactions = 10 +max_wal_senders = 10 +wal_sender_timeout = 0 +]); $node_publisher->start; # Create subscriber node. @@ -58,6 +81,7 @@ $node_subscriber->init(allows_streaming => 'logical'); $node_subscriber->append_conf( 'postgresql.conf', qq[ +max_prepared_transactions = 10 wal_retrieve_retry_interval = 2s ]); $node_subscriber->start; @@ -98,7 +122,7 @@ is($result, qq(0), 'check no subscription error'); # Create subscription. The table sync for test_tab2 on tap_sub will enter into # infinite error loop due to violating the unique constraint. $node_subscriber->safe_psql('postgres', - "CREATE SUBSCRIPTION tap_sub CONNECTION '$publisher_connstr' PUBLICATION tap_pub;" + "CREATE SUBSCRIPTION tap_sub CONNECTION '$publisher_connstr' PUBLICATION tap_pub WITH (streaming = on, two_phase = on);" ); $node_publisher->wait_for_catchup('tap_sub'); @@ -132,6 +156,11 @@ test_subscription_error( qq(duplicate key value violates unique constraint), 'error reported by the apply worker'); +confirm_transaction_stats_update( + $node_subscriber, + 'error_count = 1', + 'the error_count increment by the apply worker'); + # Check the table sync worker's error in the view. test_subscription_error( $node_subscriber, 'test_tab2', '', '', @@ -150,11 +179,83 @@ $node_subscriber->poll_query_until('postgres', $node_subscriber->poll_query_until('postgres', "SELECT count(1) > 0 FROM test_tab2"); -# There shouldn't be any errors in the view after dropping the subscription. +# Check updation of subscription worker transaction count statistics. +# COMMIT of an insertion of single record to test_tab1 +confirm_transaction_stats_update( + $node_subscriber, + 'commit_count = 1', + 'the commit_count increment by the apply worker'); + +# Some more tests for transaction stats +# PREPARE & COMMIT PREPARED +$node_publisher->safe_psql( + 'postgres', qq[ +BEGIN; +INSERT INTO test_tab1 VALUES (2); +PREPARE TRANSACTION 'gid1' +]); +$node_publisher->safe_psql('postgres', "COMMIT PREPARED 'gid1'"); +confirm_transaction_stats_update( + $node_subscriber, + 'commit_count = 2', + 'the commit_count increment by commit prepared'); + +# STREAM COMMIT +$node_publisher->safe_psql( + 'postgres', qq[ +BEGIN; +INSERT INTO test_tab1 VALUES(generate_series(1001, 2000)); +COMMIT; +]); +confirm_transaction_stats_update( + $node_subscriber, + 'commit_count = 3', + 'the commit_count increment by stream commit'); + +# STREAM PREPARE & COMMIT PREPARED +$node_publisher->safe_psql( + 'postgres', qq[ +BEGIN; +INSERT INTO test_tab1 VALUES(generate_series(2001, 3000)); +PREPARE TRANSACTION 'gid2' +]); +$node_publisher->safe_psql('postgres', "COMMIT PREPARED 'gid2'"); +confirm_transaction_stats_update( + $node_subscriber, + 'commit_count = 4', + 'the commit_count increment by streamed commit prepared'); + +# ROLLBACK PREPARED +$node_publisher->safe_psql( + 'postgres', qq[ +BEGIN; +INSERT INTO test_tab1 VALUES (3); +PREPARE TRANSACTION 'gid3'; +]); +$node_publisher->safe_psql('postgres', "ROLLBACK PREPARED 'gid3'"); +confirm_transaction_stats_update( + $node_subscriber, + 'abort_count = 1', + 'the abort_count increment by rollback prepared'); + +# STREAM PREPARE & ROLLBACK PREPARED +$node_publisher->safe_psql( + 'postgres', qq[ +BEGIN; +INSERT INTO test_tab1 VALUES (generate_series(3001, 4000)); +PREPARE TRANSACTION 'gid4'; +]); +$node_publisher->safe_psql('postgres', "ROLLBACK PREPARED 'gid4'"); +confirm_transaction_stats_update( + $node_subscriber, + 'abort_count = 2', + 'the abort_count increment by streamed rollback prepared'); + +# There shouldn't be any records in the view after dropping the subscription. $node_subscriber->safe_psql('postgres', "DROP SUBSCRIPTION tap_sub;"); $result = $node_subscriber->safe_psql('postgres', "SELECT count(1) FROM pg_stat_subscription_workers"); -is($result, q(0), 'no error after dropping subscription'); +is($result, q(0), 'no record after dropping subscription'); $node_subscriber->stop('fast'); $node_publisher->stop('fast'); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 0c61ccb..238e9fa 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -1947,6 +1947,7 @@ PgStat_MsgResetslrucounter PgStat_MsgSLRU PgStat_MsgSubscriptionPurge PgStat_MsgSubWorkerError +PgStat_MsgSubWorkerXactEnd PgStat_MsgTabpurge PgStat_MsgTabstat PgStat_MsgTempFile -- 1.8.3.1 [application/octet-stream] v17-0001-Fix-alignments-of-TAP-tests-for-pg_stat_subscrip.patch (7.4K, ../../TYCPR01MB83734A7A0596AC7ADB0DCB51ED779@TYCPR01MB8373.jpnprd01.prod.outlook.com/3-v17-0001-Fix-alignments-of-TAP-tests-for-pg_stat_subscrip.patch) download | inline diff: From 48f8d7553a1820aa54c899c245e08d9da2b51833 Mon Sep 17 00:00:00 2001 From: Takamichi Osumi <[email protected]> Date: Tue, 7 Dec 2021 07:39:58 +0000 Subject: [PATCH v17 1/2] Fix alignments of TAP tests for pg_stat_subscription_workers --- src/test/subscription/t/026_worker_stats.pl | 129 ++++++++++++++-------------- 1 file changed, 63 insertions(+), 66 deletions(-) diff --git a/src/test/subscription/t/026_worker_stats.pl b/src/test/subscription/t/026_worker_stats.pl index b662be3..8005c54 100644 --- a/src/test/subscription/t/026_worker_stats.pl +++ b/src/test/subscription/t/026_worker_stats.pl @@ -11,33 +11,37 @@ use Test::More tests => 3; # Test if the error reported on pg_stat_subscription_workers view is expected. sub test_subscription_error { - my ($node, $relname, $command, $xid, $by_apply_worker, $errmsg_prefix, $msg) - = @_; + my ($node, $relname, $command, $xid, $by_apply_worker, $errmsg_prefix, + $msg) + = @_; - my $check_sql = qq[ + my $check_sql = qq[ SELECT count(1) > 0 FROM pg_stat_subscription_workers WHERE last_error_relid = '$relname'::regclass AND starts_with(last_error_message, '$errmsg_prefix')]; - # subrelid - $check_sql .= $by_apply_worker - ? qq[ AND subrelid IS NULL] - : qq[ AND subrelid = '$relname'::regclass]; - - # last_error_command - $check_sql .= $command eq '' - ? qq[ AND last_error_command IS NULL] - : qq[ AND last_error_command = '$command']; - - # last_error_xid - $check_sql .= $xid eq '' - ? qq[ AND last_error_xid IS NULL] - : qq[ AND last_error_xid = '$xid'::xid]; - - # Wait for the particular error statistics to be reported. - $node->poll_query_until('postgres', $check_sql, -) or die "Timed out while waiting for " . $msg; + # subrelid + $check_sql .= + $by_apply_worker + ? qq[ AND subrelid IS NULL] + : qq[ AND subrelid = '$relname'::regclass]; + + # last_error_command + $check_sql .= + $command eq '' + ? qq[ AND last_error_command IS NULL] + : qq[ AND last_error_command = '$command']; + + # last_error_xid + $check_sql .= + $xid eq '' + ? qq[ AND last_error_xid IS NULL] + : qq[ AND last_error_xid = '$xid'::xid]; + + # Wait for the particular error statistics to be reported. + $node->poll_query_until('postgres', $check_sql,) + or die "Timed out while waiting for " . $msg; } # Create publisher node. @@ -51,8 +55,9 @@ $node_subscriber->init(allows_streaming => 'logical'); # The subscriber will enter an infinite error loop, so we don't want # to overflow the server log with error messages. -$node_subscriber->append_conf('postgresql.conf', - qq[ +$node_subscriber->append_conf( + 'postgresql.conf', + qq[ wal_retrieve_retry_interval = 2s ]); $node_subscriber->start; @@ -61,8 +66,8 @@ $node_subscriber->start; # create the same tables but with primary keys. Also, insert some data that # will conflict with the data replicated from publisher later. $node_publisher->safe_psql( - 'postgres', - qq[ + 'postgres', + qq[ BEGIN; CREATE TABLE test_tab1 (a int); CREATE TABLE test_tab2 (a int); @@ -71,8 +76,8 @@ INSERT INTO test_tab2 VALUES (1); COMMIT; ]); $node_subscriber->safe_psql( - 'postgres', - qq[ + 'postgres', + qq[ BEGIN; CREATE TABLE test_tab1 (a int primary key); CREATE TABLE test_tab2 (a int primary key); @@ -82,81 +87,73 @@ COMMIT; # Setup publications. my $publisher_connstr = $node_publisher->connstr . ' dbname=postgres'; -$node_publisher->safe_psql( - 'postgres', - "CREATE PUBLICATION tap_pub FOR TABLE test_tab1, test_tab2;"); +$node_publisher->safe_psql('postgres', + "CREATE PUBLICATION tap_pub FOR TABLE test_tab1, test_tab2;"); # There shouldn't be any subscription errors before starting logical replication. -my $result = $node_subscriber->safe_psql( - 'postgres', - "SELECT count(1) FROM pg_stat_subscription_workers"); +my $result = $node_subscriber->safe_psql('postgres', + "SELECT count(1) FROM pg_stat_subscription_workers"); is($result, qq(0), 'check no subscription error'); # Create subscription. The table sync for test_tab2 on tap_sub will enter into # infinite error loop due to violating the unique constraint. -$node_subscriber->safe_psql( - 'postgres', - "CREATE SUBSCRIPTION tap_sub CONNECTION '$publisher_connstr' PUBLICATION tap_pub;"); +$node_subscriber->safe_psql('postgres', + "CREATE SUBSCRIPTION tap_sub CONNECTION '$publisher_connstr' PUBLICATION tap_pub;" +); $node_publisher->wait_for_catchup('tap_sub'); # Wait for initial table sync for test_tab1 to finish. $node_subscriber->poll_query_until( - 'postgres', - qq[ + 'postgres', + qq[ SELECT count(1) = 1 FROM pg_subscription_rel WHERE srrelid = 'test_tab1'::regclass AND srsubstate in ('r', 's') ]) or die "Timed out while waiting for subscriber to synchronize data"; # Check the initial data. -$result = $node_subscriber->safe_psql( - 'postgres', - "SELECT count(a) FROM test_tab1"); +$result = + $node_subscriber->safe_psql('postgres', "SELECT count(a) FROM test_tab1"); is($result, q(1), 'check initial data are copied to subscriber'); # Insert more data to test_tab1, raising an error on the subscriber due to # violation of the unique constraint on test_tab1. my $xid = $node_publisher->safe_psql( - 'postgres', - qq[ + 'postgres', + qq[ BEGIN; INSERT INTO test_tab1 VALUES (1); SELECT pg_current_xact_id()::xid; COMMIT; ]); -test_subscription_error($node_subscriber, 'test_tab1', 'INSERT', $xid, - 1, # check apply worker error - qq(duplicate key value violates unique constraint), - 'error reported by the apply worker'); +test_subscription_error( + $node_subscriber, 'test_tab1', 'INSERT', $xid, + 1, # check apply worker error + qq(duplicate key value violates unique constraint), + 'error reported by the apply worker'); # Check the table sync worker's error in the view. -test_subscription_error($node_subscriber, 'test_tab2', '', '', - 0, # check tablesync worker error - qq(duplicate key value violates unique constraint), - 'the error reported by the table sync worker'); +test_subscription_error( + $node_subscriber, 'test_tab2', '', '', + 0, # check tablesync worker error + qq(duplicate key value violates unique constraint), + 'the error reported by the table sync worker'); # Test for resetting subscription worker statistics. # Truncate test_tab1 and test_tab2 so that applying changes and table sync can # continue, respectively. -$node_subscriber->safe_psql( - 'postgres', - "TRUNCATE test_tab1, test_tab2;"); +$node_subscriber->safe_psql('postgres', "TRUNCATE test_tab1, test_tab2;"); # Wait for the data to be replicated. -$node_subscriber->poll_query_until( - 'postgres', - "SELECT count(1) > 0 FROM test_tab1"); -$node_subscriber->poll_query_until( - 'postgres', - "SELECT count(1) > 0 FROM test_tab2"); +$node_subscriber->poll_query_until('postgres', + "SELECT count(1) > 0 FROM test_tab1"); +$node_subscriber->poll_query_until('postgres', + "SELECT count(1) > 0 FROM test_tab2"); # There shouldn't be any errors in the view after dropping the subscription. -$node_subscriber->safe_psql( - 'postgres', - "DROP SUBSCRIPTION tap_sub;"); -$result = $node_subscriber->safe_psql( - 'postgres', - "SELECT count(1) FROM pg_stat_subscription_workers"); +$node_subscriber->safe_psql('postgres', "DROP SUBSCRIPTION tap_sub;"); +$result = $node_subscriber->safe_psql('postgres', + "SELECT count(1) FROM pg_stat_subscription_workers"); is($result, q(0), 'no error after dropping subscription'); $node_subscriber->stop('fast'); -- 1.8.3.1 ^ permalink raw reply [nested|flat] 113+ messages in thread
* RE: Failed transaction statistics to measure the logical replication progress @ 2021-12-16 11:39 [email protected] <[email protected]> parent: Amit Kapila <[email protected]> 1 sibling, 0 replies; 113+ messages in thread From: [email protected] @ 2021-12-16 11:39 UTC (permalink / raw) To: 'Amit Kapila' <[email protected]>; +Cc: vignesh C <[email protected]>; Masahiko Sawada <[email protected]>; Greg Nancarrow <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]> On Monday, December 13, 2021 6:19 PM Amit Kapila <[email protected]> wrote: > On Tue, Dec 7, 2021 at 3:12 PM [email protected] > <[email protected]> wrote: > > > > Few questions and comments: > ======================== > 1. > The <structname>pg_stat_subscription_workers</structname> view will > contain > one row per subscription worker on which errors have occurred, for workers > applying logical replication changes and workers handling the initial data > - copy of the subscribed tables. The statistics entry is removed when the > - corresponding subscription is dropped. > + copy of the subscribed tables. Also, the row corresponding to the apply > + worker shows all transaction statistics of both types of workers on the > + subscription. The statistics entry is removed when the corresponding > + subscription is dropped. > > Why did you choose to show stats for both types of workers in one row? Now, the added stats show only the statistics of apply worker as we agreed. > 2. > + PGSTAT_MTYPE_SUBWORKERXACTEND, > } StatMsgType; > > I don't think we comma with the last message type. Fixed. > 3. > + Oid m_subrelid; > + > + /* necessary to determine column to increment */ LogicalRepMsgType > + m_command; > + > +} PgStat_MsgSubWorkerXactEnd; > > Is m_subrelid used in this patch? If not, why did you keep it? I think if you > choose to show separate stats for table sync and apply worker then probably it > will be used. Removed. > 4. > /* > + * Cumulative transaction statistics of subscription worker */ > + PgStat_Counter commit_count; PgStat_Counter error_count; > + PgStat_Counter abort_count; > + > > I think it is better to keep the order of columns as commit_count, abort_count, > error_count in the entire patch. Fixed. The new patch is shared in [1]. [1] - https://www.postgresql.org/message-id/TYCPR01MB83734A7A0596AC7ADB0DCB51ED779%40TYCPR01MB8373.jpnprd0... Best Regards, Takamichi Osumi ^ permalink raw reply [nested|flat] 113+ messages in thread
* RE: Failed transaction statistics to measure the logical replication progress @ 2021-12-16 11:41 [email protected] <[email protected]> parent: vignesh C <[email protected]> 0 siblings, 0 replies; 113+ messages in thread From: [email protected] @ 2021-12-16 11:41 UTC (permalink / raw) To: 'vignesh C' <[email protected]>; +Cc: Masahiko Sawada <[email protected]>; Amit Kapila <[email protected]>; Greg Nancarrow <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]> On Tuesday, December 14, 2021 12:45 AM vignesh C <[email protected]> wrote: > Thanks for the updated patch, few comments: > 1) Can we change this: > /* > + * Report the success of table sync as one commit to consolidate all > + * transaction stats into one record. > + */ > + pgstat_report_subworker_xact_end(MyLogicalRepWorker->subid, > + > LOGICAL_REP_MSG_COMMIT); > + > To: > /* Report the success of table sync */ > + pgstat_report_subworker_xact_end(MyLogicalRepWorker->subid, > + > LOGICAL_REP_MSG_COMMIT); > + This function call that the table sync worker reports an update of stats has been removed according to the recent discussion. > 2) Typo: ealier should be earlier > + /* > + * Report ealier than the call of process_syncing_tables() not > to miss an > + * increment of commit_count in case it leads to the process exit. See > + * process_syncing_tables_for_apply(). > + */ > + pgstat_report_subworker_xact_end(MyLogicalRepWorker->subid, > + > LOGICAL_REP_MSG_COMMIT); > + Thanks ! Fixed. > 3) Should we add an Assert for subwentry: > + /* > + * If this is a new error reported by table sync worker, > consolidate this > + * error count into the entry of apply worker, by swapping the stats > + * entries. > + */ > + if (OidIsValid(msg->m_subrelid)) > + subwentry = pgstat_get_subworker_entry(dbentry, > + msg->m_subid, > + > InvalidOid, true); > + subwentry->error_count++; The latest implementation doesn't require the call of pgstat_get_subworker_entry(). So, I skipped. > 4) Can we slightly change it to :We can change it: > +# Check the update of stats counters. > +confirm_transaction_stats_update( > + $node_subscriber, > + 'commit_count = 1', > + 'the commit_count increment by table sync'); > + > +confirm_transaction_stats_update( > + $node_subscriber, > + 'error_count = 1', > + 'the error_count increment by table sync'); > to: > +# Check updation of subscription worker transaction count statistics. > +confirm_transaction_stats_update( > + $node_subscriber, > + 'commit_count = 1', > + 'check table sync worker commit count is updated'); > + > +confirm_transaction_stats_update( > + $node_subscriber, > + 'error_count = 1', > + 'check table sync worker error count is updated'); I've removed the corresponding tests for table sync workers in the patch. But, I adopted the comment suggestion partly for the tests of the apply worker. On the other hand, I didn't fix the 3rd arguments of confirm_transaction_stats_update(). It needs to be a noun, because it's connected to another string "Timed out while waiting for ". in the function. See the definition of the function. The new patch v17 is shared in [1]. [1] - https://www.postgresql.org/message-id/TYCPR01MB83734A7A0596AC7ADB0DCB51ED779%40TYCPR01MB8373.jpnprd0... Best Regards, Takamichi Osumi ^ permalink raw reply [nested|flat] 113+ messages in thread
* RE: Failed transaction statistics to measure the logical replication progress @ 2021-12-16 13:21 [email protected] <[email protected]> parent: [email protected] <[email protected]> 1 sibling, 0 replies; 113+ messages in thread From: [email protected] @ 2021-12-16 13:21 UTC (permalink / raw) To: 'vignesh C' <[email protected]>; Amit Kapila <[email protected]>; +Cc: Masahiko Sawada <[email protected]>; Greg Nancarrow <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]> On Thursday, December 16, 2021 8:37 PM I wrote: > Hi, created a new version v17 according to the recent discussion with changes > to address other review comments. FYI, in v17 I've removed one part of commit message about spool file statistics on the subscriber. My intention is just to make the patches more committable shape. Although I deleted it, I'd say still there would be some room for the discussion of the necessity. It's because to begin with, we are interested in the disk writes (for the logical replication, pg_stat_replication_slots is an example), and secondly there can be a scenario that if the user of logical replication dislikes and wants to suppress unnecessary writes of file on the subscriber (STREAM ABORT causes truncate of file with changes, IIUC) they can increase the logical_decoding_work_mem on the publisher. I'll postpone this discussion, till it becomes necessary or will abandon this idea, if it's rejected. Anyway, I detached the discussion by removing it from the commit message. Best Regards, Takamichi Osumi ^ permalink raw reply [nested|flat] 113+ messages in thread
* Re: Failed transaction statistics to measure the logical replication progress @ 2021-12-17 05:03 Kyotaro Horiguchi <[email protected]> parent: [email protected] <[email protected]> 1 sibling, 1 reply; 113+ messages in thread From: Kyotaro Horiguchi @ 2021-12-17 05:03 UTC (permalink / raw) To: [email protected]; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; [email protected]; [email protected] At Thu, 16 Dec 2021 11:36:46 +0000, "[email protected]" <[email protected]> wrote in > On Thursday, December 16, 2021 4:00 PM I wrote: > > Thank you, everyone for confirming the direction. > > I'll follow the consensus of the community and fix the patch, including other > > comments. > > I'll treat only the stats for apply workers. > Hi, created a new version v17 according to the recent discussion > with changes to address other review comments. > > Kindly have a look at it. It sends stats packets at every commit-like operation on apply workers. The current pgstat is so smart that it refrain from sending stats packets at too high frequency. We already suffer frequent stats packets so apply workers need to bahave the same way. That is, the new stats numbers are once accumulated locally then the accumulated numbers are sent to stats collector by pgstat_report_stat. regards. -- Kyotaro Horiguchi NTT Open Source Software Center ^ permalink raw reply [nested|flat] 113+ messages in thread
* RE: Failed transaction statistics to measure the logical replication progress @ 2021-12-20 09:40 [email protected] <[email protected]> parent: Kyotaro Horiguchi <[email protected]> 0 siblings, 1 reply; 113+ messages in thread From: [email protected] @ 2021-12-20 09:40 UTC (permalink / raw) To: 'Kyotaro Horiguchi' <[email protected]>; +Cc: [email protected] <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]> Friday, December 17, 2021 2:03 PM Kyotaro Horiguchi <[email protected]> wrote: > It sends stats packets at every commit-like operation on apply workers. The > current pgstat is so smart that it refrain from sending stats packets at too high > frequency. We already suffer frequent stats packets so apply workers need to > bahave the same way. > > That is, the new stats numbers are once accumulated locally then the > accumulated numbers are sent to stats collector by pgstat_report_stat. Hi, Horiguchi-san. I felt your point is absolutely right ! Updated the patch to address your concern. Best Regards, Takamichi Osumi Attachments: [application/octet-stream] v18-0002-Extend-pg_stat_subscription_workers-to-include-g.patch (27.3K, ../../TYCPR01MB83736371B66F90C71365ED11ED7B9@TYCPR01MB8373.jpnprd01.prod.outlook.com/2-v18-0002-Extend-pg_stat_subscription_workers-to-include-g.patch) download | inline diff: From c1ea66e5028e74403a1ef33e9c7c7ea3bb9786fd Mon Sep 17 00:00:00 2001 From: Takamichi Osumi <[email protected]> Date: Mon, 20 Dec 2021 09:13:08 +0000 Subject: [PATCH v18 2/2] Extend pg_stat_subscription_workers to include general transaction statistics Categorize transactions of logical replication subscriber into three types (commit, abort, error) and introduce cumulative columns of those numbers in the pg_stat_subscription_workers. In order to avoid having a large number of entries to be created by the table synchronization for many tables, the new stats columns are utilized only by the apply worker. Author: Takamichi Osumi Reviewed-by: Amit Kapila, Masahiko Sawada, Hou Zhijie, Greg Nancarrow, Vignesh C, Ajin Cherian, Kyotaro Horiguchi Discussion: https://www.postgresql.org/message-id/OSBPR01MB48887CA8F40C8D984A6DC00CED199%40OSBPR01MB4888.jpnprd01.prod.outlook.com --- doc/src/sgml/monitoring.sgml | 42 +++++++++- src/backend/catalog/system_views.sql | 3 + src/backend/postmaster/pgstat.c | 124 ++++++++++++++++++++++++++++ src/backend/replication/logical/launcher.c | 4 + src/backend/replication/logical/worker.c | 22 +++++ src/backend/utils/adt/pgstatfuncs.c | 25 ++++-- src/include/catalog/pg_proc.dat | 6 +- src/include/pgstat.h | 32 +++++++ src/include/replication/worker_internal.h | 6 ++ src/test/regress/expected/rules.out | 5 +- src/test/subscription/t/026_worker_stats.pl | 109 +++++++++++++++++++++++- src/tools/pgindent/typedefs.list | 1 + 12 files changed, 360 insertions(+), 19 deletions(-) diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml index 62f2a33..e795a1d 100644 --- a/doc/src/sgml/monitoring.sgml +++ b/doc/src/sgml/monitoring.sgml @@ -629,8 +629,8 @@ postgres 27093 0.0 0.0 30096 2752 ? Ss 11:34 0:00 postgres: ser <row> <entry><structname>pg_stat_subscription_workers</structname><indexterm><primary>pg_stat_subscription_workers</primary></indexterm></entry> - <entry>One row per subscription worker, showing statistics about errors - that occurred on that subscription worker. + <entry>One row per subscription worker, showing statistics about transactions + and errors that occurred on that subscription worker. See <link linkend="monitoring-pg-stat-subscription-workers"> <structname>pg_stat_subscription_workers</structname></link> for details. </entry> @@ -3074,8 +3074,10 @@ SELECT pid, wait_event_type, wait_event FROM pg_stat_activity WHERE wait_event i The <structname>pg_stat_subscription_workers</structname> view will contain one row per subscription worker on which errors have occurred, for workers applying logical replication changes and workers handling the initial data - copy of the subscribed tables. The statistics entry is removed when the - corresponding subscription is dropped. + copy of the subscribed tables. The row corresponding to the apply + worker shows transaction statistics of the main apply worker on the + subscription. The statistics entry is removed when the corresponding + subscription is dropped. </para> <table id="pg-stat-subscription-workers" xreflabel="pg_stat_subscription_workers"> @@ -3123,6 +3125,38 @@ SELECT pid, wait_event_type, wait_event FROM pg_stat_activity WHERE wait_event i <row> <entry role="catalog_table_entry"><para role="column_definition"> + <structfield>commit_count</structfield> <type>bigint</type> + </para> + <para> + Number of transactions successfully applied in this subscription. + Both COMMIT and COMMIT PREPARED increment this counter. + </para></entry> + </row> + + <row> + <entry role="catalog_table_entry"><para role="column_definition"> + <structfield>abort_count</structfield> <type>bigint</type> + </para> + <para> + Number of transactions aborted in this subscription. + ROLLBACK PREPARED increments this counter. + </para></entry> + </row> + + <row> + <entry role="catalog_table_entry"><para role="column_definition"> + <structfield>error_count</structfield> <type>bigint</type> + </para> + <para> + Number of transactions that failed to be applied by the table + sync worker or main apply worker in this subscription. This + counter is updated after confirming the error is not same as + the previous one. + </para></entry> + </row> + + <row> + <entry role="catalog_table_entry"><para role="column_definition"> <structfield>last_error_relid</structfield> <type>oid</type> </para> <para> diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index 61b515c..dda3cd7 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1267,6 +1267,9 @@ CREATE VIEW pg_stat_subscription_workers AS w.subid, s.subname, w.subrelid, + w.commit_count, + w.abort_count, + w.error_count, w.last_error_relid, w.last_error_command, w.last_error_xid, diff --git a/src/backend/postmaster/pgstat.c b/src/backend/postmaster/pgstat.c index 7264d2c..927c187 100644 --- a/src/backend/postmaster/pgstat.c +++ b/src/backend/postmaster/pgstat.c @@ -382,6 +382,7 @@ static void pgstat_recv_replslot(PgStat_MsgReplSlot *msg, int len); static void pgstat_recv_tempfile(PgStat_MsgTempFile *msg, int len); static void pgstat_recv_subscription_purge(PgStat_MsgSubscriptionPurge *msg, int len); static void pgstat_recv_subworker_error(PgStat_MsgSubWorkerError *msg, int len); +static void pgstat_recv_subworker_xact_end(PgStat_MsgSubWorkerXactEnd *msg, int len); /* ------------------------------------------------------------ * Public functions called from postmaster follow @@ -1949,6 +1950,42 @@ pgstat_report_replslot_drop(const char *slotname) } /* ---------- + * pgstat_report_subworker_xact_end() - + * + * Tell the collector that worker transaction has successfully completed. + * This should be called before the call of process_syning_tables() not to + * miss an increment of transaction stats in case it leads to a process exit. + * See process_syncing_tables_for_apply(). + * ---------- + */ +void +pgstat_report_subworker_xact_end(LogicalRepWorker *repWorker, + LogicalRepMsgType command, bool force) +{ + Assert(command == LOGICAL_REP_MSG_COMMIT || + command == LOGICAL_REP_MSG_STREAM_COMMIT || + command == LOGICAL_REP_MSG_COMMIT_PREPARED || + command == LOGICAL_REP_MSG_ROLLBACK_PREPARED); + + switch (command) + { + case LOGICAL_REP_MSG_COMMIT: + case LOGICAL_REP_MSG_STREAM_COMMIT: + case LOGICAL_REP_MSG_COMMIT_PREPARED: + repWorker->commit_count++; + break; + case LOGICAL_REP_MSG_ROLLBACK_PREPARED: + repWorker->abort_count++; + break; + default: + elog(ERROR, "unexpected logical message type as transaction end"); + break; + } + + pgstat_send_subworker_xact_stats(repWorker, force); +} + +/* ---------- * pgstat_report_subworker_error() - * * Tell the collector about the subscription worker error. @@ -3484,6 +3521,58 @@ pgstat_send_subscription_purge(PgStat_MsgSubscriptionPurge *msg) } /* ---------- + * pgstat_send_subworker_xact_stats() - + * + * Send a subworker transaction stats to the collector. + * 'force' becomes true only when the subscription worker process exits. + * ---------- + */ +void +pgstat_send_subworker_xact_stats(LogicalRepWorker *repWorker, bool force) +{ + static TimestampTz last_report = 0; + PgStat_MsgSubWorkerXactEnd msg; + + if (!force) + { + TimestampTz now = GetCurrentTimestamp(); + + /* + * Don't send a message unless it's been at least PGSTAT_STAT_INTERVAL + * msec since we last sent one to avoid overloading the stats + * collector. + */ + if (!TimestampDifferenceExceeds(last_report, now, PGSTAT_STAT_INTERVAL)) + return; + last_report = now; + } + + /* + * This function can be called even if nothing at all has happened. In + * this case, avoid sending a completely empty message to the stats + * collector. + */ + if (repWorker->commit_count == 0 && repWorker->abort_count == 0) + return; + + /* + * Prepare and send the message + */ + pgstat_setheader(&msg.m_hdr, PGSTAT_MTYPE_SUBWORKERXACTEND); + msg.m_databaseid = MyDatabaseId; + msg.m_subid = repWorker->subid; + msg.commit_count = repWorker->commit_count; + msg.abort_count = repWorker->abort_count; + pgstat_send(&msg, sizeof(PgStat_MsgSubWorkerXactEnd)); + + /* + * Clear out the statistics buffer, so it can be re-used. + */ + repWorker->commit_count = 0; + repWorker->abort_count = 0; +} + +/* ---------- * PgstatCollectorMain() - * * Start up the statistics collector process. This is the body of the @@ -3746,6 +3835,10 @@ PgstatCollectorMain(int argc, char *argv[]) pgstat_recv_subworker_error(&msg.msg_subworkererror, len); break; + case PGSTAT_MTYPE_SUBWORKERXACTEND: + pgstat_recv_subworker_xact_end(&msg.msg_subworkerxactend, len); + break; + default: break; } @@ -3965,6 +4058,9 @@ pgstat_get_subworker_entry(PgStat_StatDBEntry *dbentry, Oid subid, Oid subrelid, /* If not found, initialize the new one */ if (!found) { + subwentry->commit_count = 0; + subwentry->abort_count = 0; + subwentry->error_count = 0; subwentry->last_error_relid = InvalidOid; subwentry->last_error_command = 0; subwentry->last_error_xid = InvalidTransactionId; @@ -6153,6 +6249,27 @@ pgstat_recv_subscription_purge(PgStat_MsgSubscriptionPurge *msg, int len) } /* ---------- + * pgstat_recv_subworker_xact_end() - + * + * Process a SUBWORKERXACTEND message. + * ---------- + */ +static void +pgstat_recv_subworker_xact_end(PgStat_MsgSubWorkerXactEnd *msg, int len) +{ + PgStat_StatDBEntry *dbentry; + PgStat_StatSubWorkerEntry *wentry; + + dbentry = pgstat_get_db_entry(msg->m_databaseid, true); + wentry = pgstat_get_subworker_entry(dbentry, msg->m_subid, + InvalidOid, true); + Assert(wentry); + + wentry->commit_count += msg->commit_count; + wentry->abort_count += msg->abort_count; +} + +/* ---------- * pgstat_recv_subworker_error() - * * Process a SUBWORKERERROR message. @@ -6193,6 +6310,13 @@ pgstat_recv_subworker_error(PgStat_MsgSubWorkerError *msg, int len) subwentry->last_error_time = msg->m_timestamp; strlcpy(subwentry->last_error_message, msg->m_message, PGSTAT_SUBWORKERERROR_MSGLEN); + + /* + * Only if this is a new error reported by the apply worker, increment the + * counter of error. + */ + if (!OidIsValid(msg->m_subrelid)) + subwentry->error_count++; } /* ---------- diff --git a/src/backend/replication/logical/launcher.c b/src/backend/replication/logical/launcher.c index 3fb4caa..bb67c66 100644 --- a/src/backend/replication/logical/launcher.c +++ b/src/backend/replication/logical/launcher.c @@ -385,6 +385,8 @@ retry: TIMESTAMP_NOBEGIN(worker->last_recv_time); worker->reply_lsn = InvalidXLogRecPtr; TIMESTAMP_NOBEGIN(worker->reply_time); + worker->commit_count = 0; + worker->abort_count = 0; /* Before releasing lock, remember generation for future identification. */ generation = worker->generation; @@ -647,6 +649,8 @@ logicalrep_worker_onexit(int code, Datum arg) if (LogRepWorkerWalRcvConn) walrcv_disconnect(LogRepWorkerWalRcvConn); + pgstat_send_subworker_xact_stats(MyLogicalRepWorker, true); + logicalrep_worker_detach(); /* Cleanup fileset used for streaming transactions. */ diff --git a/src/backend/replication/logical/worker.c b/src/backend/replication/logical/worker.c index 2e79302..7bcf052 100644 --- a/src/backend/replication/logical/worker.c +++ b/src/backend/replication/logical/worker.c @@ -815,6 +815,10 @@ apply_handle_commit(StringInfo s) apply_handle_commit_internal(&commit_data); + /* Update stats */ + pgstat_report_subworker_xact_end(MyLogicalRepWorker, + LOGICAL_REP_MSG_COMMIT, false); + /* Process any tables that are being synchronized in parallel. */ process_syncing_tables(commit_data.end_lsn); @@ -960,6 +964,10 @@ apply_handle_commit_prepared(StringInfo s) store_flush_position(prepare_data.end_lsn); in_remote_transaction = false; + /* Update stats */ + pgstat_report_subworker_xact_end(MyLogicalRepWorker, + LOGICAL_REP_MSG_COMMIT_PREPARED, false); + /* Process any tables that are being synchronized in parallel. */ process_syncing_tables(prepare_data.end_lsn); @@ -1011,6 +1019,10 @@ apply_handle_rollback_prepared(StringInfo s) store_flush_position(rollback_data.rollback_end_lsn); in_remote_transaction = false; + /* Update stats */ + pgstat_report_subworker_xact_end(MyLogicalRepWorker, + LOGICAL_REP_MSG_ROLLBACK_PREPARED, false); + /* Process any tables that are being synchronized in parallel. */ process_syncing_tables(rollback_data.rollback_end_lsn); @@ -1435,6 +1447,10 @@ apply_handle_stream_commit(StringInfo s) /* unlink the files with serialized changes and subxact info */ stream_cleanup_files(MyLogicalRepWorker->subid, xid); + /* Update stats */ + pgstat_report_subworker_xact_end(MyLogicalRepWorker, + LOGICAL_REP_MSG_STREAM_COMMIT, false); + /* Process any tables that are being synchronized in parallel. */ process_syncing_tables(commit_data.end_lsn); @@ -2561,6 +2577,12 @@ LogicalRepApplyLoop(XLogRecPtr last_received) MemoryContextSwitchTo(ApplyMessageContext); + /* + * Process if we have pending transaction stats and reached + * PGSTAT_STAT_INTERVAL + */ + pgstat_send_subworker_xact_stats(MyLogicalRepWorker, false); + len = walrcv_receive(LogRepWorkerWalRcvConn, &buf, &fd); if (len != 0) diff --git a/src/backend/utils/adt/pgstatfuncs.c b/src/backend/utils/adt/pgstatfuncs.c index f529c15..87b5254 100644 --- a/src/backend/utils/adt/pgstatfuncs.c +++ b/src/backend/utils/adt/pgstatfuncs.c @@ -2415,7 +2415,7 @@ pg_stat_get_replication_slot(PG_FUNCTION_ARGS) Datum pg_stat_get_subscription_worker(PG_FUNCTION_ARGS) { -#define PG_STAT_GET_SUBSCRIPTION_WORKER_COLS 8 +#define PG_STAT_GET_SUBSCRIPTION_WORKER_COLS 11 Oid subid = PG_GETARG_OID(0); Oid subrelid; TupleDesc tupdesc; @@ -2442,17 +2442,23 @@ pg_stat_get_subscription_worker(PG_FUNCTION_ARGS) OIDOID, -1, 0); TupleDescInitEntry(tupdesc, (AttrNumber) 2, "subrelid", OIDOID, -1, 0); - TupleDescInitEntry(tupdesc, (AttrNumber) 3, "last_error_relid", + TupleDescInitEntry(tupdesc, (AttrNumber) 3, "commit_count", + INT8OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) 4, "abort_count", + INT8OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) 5, "error_count", + INT8OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) 6, "last_error_relid", OIDOID, -1, 0); - TupleDescInitEntry(tupdesc, (AttrNumber) 4, "last_error_command", + TupleDescInitEntry(tupdesc, (AttrNumber) 7, "last_error_command", TEXTOID, -1, 0); - TupleDescInitEntry(tupdesc, (AttrNumber) 5, "last_error_xid", + TupleDescInitEntry(tupdesc, (AttrNumber) 8, "last_error_xid", XIDOID, -1, 0); - TupleDescInitEntry(tupdesc, (AttrNumber) 6, "last_error_count", + TupleDescInitEntry(tupdesc, (AttrNumber) 9, "last_error_count", INT8OID, -1, 0); - TupleDescInitEntry(tupdesc, (AttrNumber) 7, "last_error_message", + TupleDescInitEntry(tupdesc, (AttrNumber) 10, "last_error_message", TEXTOID, -1, 0); - TupleDescInitEntry(tupdesc, (AttrNumber) 8, "last_error_time", + TupleDescInitEntry(tupdesc, (AttrNumber) 11, "last_error_time", TIMESTAMPTZOID, -1, 0); BlessTupleDesc(tupdesc); @@ -2470,6 +2476,11 @@ pg_stat_get_subscription_worker(PG_FUNCTION_ARGS) else nulls[i++] = true; + /* transaction stats */ + values[i++] = Int64GetDatum(wentry->commit_count); + values[i++] = Int64GetDatum(wentry->abort_count); + values[i++] = Int64GetDatum(wentry->error_count); + /* last_error_relid */ if (OidIsValid(wentry->last_error_relid)) values[i++] = ObjectIdGetDatum(wentry->last_error_relid); diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index 4d992dc..b76f84a 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -5375,9 +5375,9 @@ proname => 'pg_stat_get_subscription_worker', prorows => '1', proisstrict => 'f', proretset => 't', provolatile => 's', proparallel => 'r', prorettype => 'record', proargtypes => 'oid oid', - proallargtypes => '{oid,oid,oid,oid,oid,text,xid,int8,text,timestamptz}', - proargmodes => '{i,i,o,o,o,o,o,o,o,o}', - proargnames => '{subid,subrelid,subid,subrelid,last_error_relid,last_error_command,last_error_xid,last_error_count,last_error_message,last_error_time}', + proallargtypes => '{oid,oid,oid,oid,int8,int8,int8,oid,text,xid,int8,text,timestamptz}', + proargmodes => '{i,i,o,o,o,o,o,o,o,o,o,o,o}', + proargnames => '{subid,subrelid,subid,subrelid,commit_count,abort_count,error_count,last_error_relid,last_error_command,last_error_xid,last_error_count,last_error_message,last_error_time}', prosrc => 'pg_stat_get_subscription_worker' }, { oid => '6118', descr => 'statistics: information about subscription', proname => 'pg_stat_get_subscription', prorows => '10', proisstrict => 'f', diff --git a/src/include/pgstat.h b/src/include/pgstat.h index 5b51b58..be55b55 100644 --- a/src/include/pgstat.h +++ b/src/include/pgstat.h @@ -15,6 +15,7 @@ #include "portability/instr_time.h" #include "postmaster/pgarch.h" /* for MAX_XFN_CHARS */ #include "replication/logicalproto.h" +#include "replication/worker_internal.h" #include "utils/backend_progress.h" /* for backward compatibility */ #include "utils/backend_status.h" /* for backward compatibility */ #include "utils/hsearch.h" @@ -86,6 +87,7 @@ typedef enum StatMsgType PGSTAT_MTYPE_DISCONNECT, PGSTAT_MTYPE_SUBSCRIPTIONPURGE, PGSTAT_MTYPE_SUBWORKERERROR, + PGSTAT_MTYPE_SUBWORKERXACTEND } StatMsgType; /* ---------- @@ -558,6 +560,23 @@ typedef struct PgStat_MsgSubscriptionPurge } PgStat_MsgSubscriptionPurge; /* ---------- + * PgStat_MsgSubscriptionXactEnd Sent by the apply worker to report transaction + * ends. + * ---------- + */ +typedef struct PgStat_MsgSubWorkerXactEnd +{ + PgStat_MsgHdr m_hdr; + + /* determine the worker entry */ + Oid m_databaseid; + Oid m_subid; + + PgStat_Counter commit_count; + PgStat_Counter abort_count; +} PgStat_MsgSubWorkerXactEnd; + +/* ---------- * PgStat_MsgSubWorkerError Sent by the apply worker or the table sync * worker to report the error occurred while * processing changes. @@ -769,6 +788,7 @@ typedef union PgStat_Msg PgStat_MsgDisconnect msg_disconnect; PgStat_MsgSubscriptionPurge msg_subscriptionpurge; PgStat_MsgSubWorkerError msg_subworkererror; + PgStat_MsgSubWorkerXactEnd msg_subworkerxactend; } PgStat_Msg; @@ -1010,6 +1030,13 @@ typedef struct PgStat_StatSubWorkerEntry PgStat_StatSubWorkerKey key; /* hash key (must be first) */ /* + * Cumulative transaction statistics of subscription worker + */ + PgStat_Counter commit_count; + PgStat_Counter abort_count; + PgStat_Counter error_count; + + /* * Subscription worker error statistics representing an error that * occurred during application of changes or the initial table * synchronization. @@ -1131,6 +1158,9 @@ extern void pgstat_report_checksum_failure(void); extern void pgstat_report_replslot(const PgStat_StatReplSlotEntry *repSlotStat); extern void pgstat_report_replslot_create(const char *slotname); extern void pgstat_report_replslot_drop(const char *slotname); +extern void pgstat_report_subworker_xact_end(LogicalRepWorker *repWorker, + LogicalRepMsgType command, + bool bforce); extern void pgstat_report_subworker_error(Oid subid, Oid subrelid, Oid relid, LogicalRepMsgType command, TransactionId xid, const char *errmsg); @@ -1217,6 +1247,8 @@ extern void pgstat_send_archiver(const char *xlog, bool failed); extern void pgstat_send_bgwriter(void); extern void pgstat_send_checkpointer(void); extern void pgstat_send_wal(bool force); +extern void pgstat_send_subworker_xact_stats(LogicalRepWorker *repWorker, + bool force); /* ---------- * Support functions for the SQL-callable functions to diff --git a/src/include/replication/worker_internal.h b/src/include/replication/worker_internal.h index 9d29849..d6e4570 100644 --- a/src/include/replication/worker_internal.h +++ b/src/include/replication/worker_internal.h @@ -66,6 +66,12 @@ typedef struct LogicalRepWorker TimestampTz last_recv_time; XLogRecPtr reply_lsn; TimestampTz reply_time; + + /* + * Transaction statistics of subscription worker + */ + int64 commit_count; + int64 abort_count; } LogicalRepWorker; /* Main memory context for apply worker. Permanent during worker lifetime. */ diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out index b58b062..2b0bfae 100644 --- a/src/test/regress/expected/rules.out +++ b/src/test/regress/expected/rules.out @@ -2097,6 +2097,9 @@ pg_stat_subscription| SELECT su.oid AS subid, pg_stat_subscription_workers| SELECT w.subid, s.subname, w.subrelid, + w.commit_count, + w.abort_count, + w.error_count, w.last_error_relid, w.last_error_command, w.last_error_xid, @@ -2110,7 +2113,7 @@ pg_stat_subscription_workers| SELECT w.subid, SELECT pg_subscription_rel.srsubid AS subid, pg_subscription_rel.srrelid AS relid FROM pg_subscription_rel) sr, - (LATERAL pg_stat_get_subscription_worker(sr.subid, sr.relid) w(subid, subrelid, last_error_relid, last_error_command, last_error_xid, last_error_count, last_error_message, last_error_time) + (LATERAL pg_stat_get_subscription_worker(sr.subid, sr.relid) w(subid, subrelid, commit_count, abort_count, error_count, last_error_relid, last_error_command, last_error_xid, last_error_count, last_error_message, last_error_time) JOIN pg_subscription s ON ((w.subid = s.oid))); pg_stat_sys_indexes| SELECT pg_stat_all_indexes.relid, pg_stat_all_indexes.indexrelid, diff --git a/src/test/subscription/t/026_worker_stats.pl b/src/test/subscription/t/026_worker_stats.pl index 8005c54..99253f4 100644 --- a/src/test/subscription/t/026_worker_stats.pl +++ b/src/test/subscription/t/026_worker_stats.pl @@ -1,7 +1,7 @@ # Copyright (c) 2021, PostgreSQL Global Development Group -# Tests for subscription error stats. +# Tests for subscription stats. use strict; use warnings; use PostgreSQL::Test::Cluster; @@ -44,9 +44,32 @@ WHERE last_error_relid = '$relname'::regclass or die "Timed out while waiting for " . $msg; } +# Test whether the update of general transaction stats satisfies the expected +# condition or not. +sub confirm_transaction_stats_update +{ + my ($node, $condition, $msg) = @_; + + # Check only the stats of the apply worker + my $sql = qq[ +SELECT count(1) = 1 +FROM pg_stat_subscription_workers +WHERE subrelid IS NULL AND $condition]; + + $node->poll_query_until('postgres', $sql) + or die "Timed out while waiting for " . $msg; +} + # Create publisher node. my $node_publisher = PostgreSQL::Test::Cluster->new('publisher'); $node_publisher->init(allows_streaming => 'logical'); +$node_publisher->append_conf( + 'postgresql.conf', qq[ +logical_decoding_work_mem = 64kB +max_prepared_transactions = 10 +max_wal_senders = 10 +wal_sender_timeout = 0 +]); $node_publisher->start; # Create subscriber node. @@ -58,6 +81,7 @@ $node_subscriber->init(allows_streaming => 'logical'); $node_subscriber->append_conf( 'postgresql.conf', qq[ +max_prepared_transactions = 10 wal_retrieve_retry_interval = 2s ]); $node_subscriber->start; @@ -98,7 +122,7 @@ is($result, qq(0), 'check no subscription error'); # Create subscription. The table sync for test_tab2 on tap_sub will enter into # infinite error loop due to violating the unique constraint. $node_subscriber->safe_psql('postgres', - "CREATE SUBSCRIPTION tap_sub CONNECTION '$publisher_connstr' PUBLICATION tap_pub;" + "CREATE SUBSCRIPTION tap_sub CONNECTION '$publisher_connstr' PUBLICATION tap_pub WITH (streaming = on, two_phase = on);" ); $node_publisher->wait_for_catchup('tap_sub'); @@ -132,6 +156,11 @@ test_subscription_error( qq(duplicate key value violates unique constraint), 'error reported by the apply worker'); +confirm_transaction_stats_update( + $node_subscriber, + 'error_count = 1', + 'the error_count increment by the apply worker'); + # Check the table sync worker's error in the view. test_subscription_error( $node_subscriber, 'test_tab2', '', '', @@ -150,11 +179,83 @@ $node_subscriber->poll_query_until('postgres', $node_subscriber->poll_query_until('postgres', "SELECT count(1) > 0 FROM test_tab2"); -# There shouldn't be any errors in the view after dropping the subscription. +# Check updation of subscription worker transaction count statistics. +# COMMIT of an insertion of single record to test_tab1 +confirm_transaction_stats_update( + $node_subscriber, + 'commit_count = 1', + 'the commit_count increment by the apply worker'); + +# Some more tests for transaction stats +# PREPARE & COMMIT PREPARED +$node_publisher->safe_psql( + 'postgres', qq[ +BEGIN; +INSERT INTO test_tab1 VALUES (2); +PREPARE TRANSACTION 'gid1' +]); +$node_publisher->safe_psql('postgres', "COMMIT PREPARED 'gid1'"); +confirm_transaction_stats_update( + $node_subscriber, + 'commit_count = 2', + 'the commit_count increment by commit prepared'); + +# STREAM COMMIT +$node_publisher->safe_psql( + 'postgres', qq[ +BEGIN; +INSERT INTO test_tab1 VALUES(generate_series(1001, 2000)); +COMMIT; +]); +confirm_transaction_stats_update( + $node_subscriber, + 'commit_count = 3', + 'the commit_count increment by stream commit'); + +# STREAM PREPARE & COMMIT PREPARED +$node_publisher->safe_psql( + 'postgres', qq[ +BEGIN; +INSERT INTO test_tab1 VALUES(generate_series(2001, 3000)); +PREPARE TRANSACTION 'gid2' +]); +$node_publisher->safe_psql('postgres', "COMMIT PREPARED 'gid2'"); +confirm_transaction_stats_update( + $node_subscriber, + 'commit_count = 4', + 'the commit_count increment by streamed commit prepared'); + +# ROLLBACK PREPARED +$node_publisher->safe_psql( + 'postgres', qq[ +BEGIN; +INSERT INTO test_tab1 VALUES (3); +PREPARE TRANSACTION 'gid3'; +]); +$node_publisher->safe_psql('postgres', "ROLLBACK PREPARED 'gid3'"); +confirm_transaction_stats_update( + $node_subscriber, + 'abort_count = 1', + 'the abort_count increment by rollback prepared'); + +# STREAM PREPARE & ROLLBACK PREPARED +$node_publisher->safe_psql( + 'postgres', qq[ +BEGIN; +INSERT INTO test_tab1 VALUES (generate_series(3001, 4000)); +PREPARE TRANSACTION 'gid4'; +]); +$node_publisher->safe_psql('postgres', "ROLLBACK PREPARED 'gid4'"); +confirm_transaction_stats_update( + $node_subscriber, + 'abort_count = 2', + 'the abort_count increment by streamed rollback prepared'); + +# There shouldn't be any records in the view after dropping the subscription. $node_subscriber->safe_psql('postgres', "DROP SUBSCRIPTION tap_sub;"); $result = $node_subscriber->safe_psql('postgres', "SELECT count(1) FROM pg_stat_subscription_workers"); -is($result, q(0), 'no error after dropping subscription'); +is($result, q(0), 'no record after dropping subscription'); $node_subscriber->stop('fast'); $node_publisher->stop('fast'); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 0c61ccb..238e9fa 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -1947,6 +1947,7 @@ PgStat_MsgResetslrucounter PgStat_MsgSLRU PgStat_MsgSubscriptionPurge PgStat_MsgSubWorkerError +PgStat_MsgSubWorkerXactEnd PgStat_MsgTabpurge PgStat_MsgTabstat PgStat_MsgTempFile -- 1.8.3.1 [application/octet-stream] v18-0001-Fix-alignments-of-TAP-tests-for-pg_stat_subscrip.patch (7.4K, ../../TYCPR01MB83736371B66F90C71365ED11ED7B9@TYCPR01MB8373.jpnprd01.prod.outlook.com/3-v18-0001-Fix-alignments-of-TAP-tests-for-pg_stat_subscrip.patch) download | inline diff: From 4b8afdb953c64d202f07a57068d249f29dc2aed0 Mon Sep 17 00:00:00 2001 From: Takamichi Osumi <[email protected]> Date: Tue, 7 Dec 2021 07:39:58 +0000 Subject: [PATCH v18 1/2] Fix alignments of TAP tests for pg_stat_subscription_workers --- src/test/subscription/t/026_worker_stats.pl | 129 ++++++++++++++-------------- 1 file changed, 63 insertions(+), 66 deletions(-) diff --git a/src/test/subscription/t/026_worker_stats.pl b/src/test/subscription/t/026_worker_stats.pl index b662be3..8005c54 100644 --- a/src/test/subscription/t/026_worker_stats.pl +++ b/src/test/subscription/t/026_worker_stats.pl @@ -11,33 +11,37 @@ use Test::More tests => 3; # Test if the error reported on pg_stat_subscription_workers view is expected. sub test_subscription_error { - my ($node, $relname, $command, $xid, $by_apply_worker, $errmsg_prefix, $msg) - = @_; + my ($node, $relname, $command, $xid, $by_apply_worker, $errmsg_prefix, + $msg) + = @_; - my $check_sql = qq[ + my $check_sql = qq[ SELECT count(1) > 0 FROM pg_stat_subscription_workers WHERE last_error_relid = '$relname'::regclass AND starts_with(last_error_message, '$errmsg_prefix')]; - # subrelid - $check_sql .= $by_apply_worker - ? qq[ AND subrelid IS NULL] - : qq[ AND subrelid = '$relname'::regclass]; - - # last_error_command - $check_sql .= $command eq '' - ? qq[ AND last_error_command IS NULL] - : qq[ AND last_error_command = '$command']; - - # last_error_xid - $check_sql .= $xid eq '' - ? qq[ AND last_error_xid IS NULL] - : qq[ AND last_error_xid = '$xid'::xid]; - - # Wait for the particular error statistics to be reported. - $node->poll_query_until('postgres', $check_sql, -) or die "Timed out while waiting for " . $msg; + # subrelid + $check_sql .= + $by_apply_worker + ? qq[ AND subrelid IS NULL] + : qq[ AND subrelid = '$relname'::regclass]; + + # last_error_command + $check_sql .= + $command eq '' + ? qq[ AND last_error_command IS NULL] + : qq[ AND last_error_command = '$command']; + + # last_error_xid + $check_sql .= + $xid eq '' + ? qq[ AND last_error_xid IS NULL] + : qq[ AND last_error_xid = '$xid'::xid]; + + # Wait for the particular error statistics to be reported. + $node->poll_query_until('postgres', $check_sql,) + or die "Timed out while waiting for " . $msg; } # Create publisher node. @@ -51,8 +55,9 @@ $node_subscriber->init(allows_streaming => 'logical'); # The subscriber will enter an infinite error loop, so we don't want # to overflow the server log with error messages. -$node_subscriber->append_conf('postgresql.conf', - qq[ +$node_subscriber->append_conf( + 'postgresql.conf', + qq[ wal_retrieve_retry_interval = 2s ]); $node_subscriber->start; @@ -61,8 +66,8 @@ $node_subscriber->start; # create the same tables but with primary keys. Also, insert some data that # will conflict with the data replicated from publisher later. $node_publisher->safe_psql( - 'postgres', - qq[ + 'postgres', + qq[ BEGIN; CREATE TABLE test_tab1 (a int); CREATE TABLE test_tab2 (a int); @@ -71,8 +76,8 @@ INSERT INTO test_tab2 VALUES (1); COMMIT; ]); $node_subscriber->safe_psql( - 'postgres', - qq[ + 'postgres', + qq[ BEGIN; CREATE TABLE test_tab1 (a int primary key); CREATE TABLE test_tab2 (a int primary key); @@ -82,81 +87,73 @@ COMMIT; # Setup publications. my $publisher_connstr = $node_publisher->connstr . ' dbname=postgres'; -$node_publisher->safe_psql( - 'postgres', - "CREATE PUBLICATION tap_pub FOR TABLE test_tab1, test_tab2;"); +$node_publisher->safe_psql('postgres', + "CREATE PUBLICATION tap_pub FOR TABLE test_tab1, test_tab2;"); # There shouldn't be any subscription errors before starting logical replication. -my $result = $node_subscriber->safe_psql( - 'postgres', - "SELECT count(1) FROM pg_stat_subscription_workers"); +my $result = $node_subscriber->safe_psql('postgres', + "SELECT count(1) FROM pg_stat_subscription_workers"); is($result, qq(0), 'check no subscription error'); # Create subscription. The table sync for test_tab2 on tap_sub will enter into # infinite error loop due to violating the unique constraint. -$node_subscriber->safe_psql( - 'postgres', - "CREATE SUBSCRIPTION tap_sub CONNECTION '$publisher_connstr' PUBLICATION tap_pub;"); +$node_subscriber->safe_psql('postgres', + "CREATE SUBSCRIPTION tap_sub CONNECTION '$publisher_connstr' PUBLICATION tap_pub;" +); $node_publisher->wait_for_catchup('tap_sub'); # Wait for initial table sync for test_tab1 to finish. $node_subscriber->poll_query_until( - 'postgres', - qq[ + 'postgres', + qq[ SELECT count(1) = 1 FROM pg_subscription_rel WHERE srrelid = 'test_tab1'::regclass AND srsubstate in ('r', 's') ]) or die "Timed out while waiting for subscriber to synchronize data"; # Check the initial data. -$result = $node_subscriber->safe_psql( - 'postgres', - "SELECT count(a) FROM test_tab1"); +$result = + $node_subscriber->safe_psql('postgres', "SELECT count(a) FROM test_tab1"); is($result, q(1), 'check initial data are copied to subscriber'); # Insert more data to test_tab1, raising an error on the subscriber due to # violation of the unique constraint on test_tab1. my $xid = $node_publisher->safe_psql( - 'postgres', - qq[ + 'postgres', + qq[ BEGIN; INSERT INTO test_tab1 VALUES (1); SELECT pg_current_xact_id()::xid; COMMIT; ]); -test_subscription_error($node_subscriber, 'test_tab1', 'INSERT', $xid, - 1, # check apply worker error - qq(duplicate key value violates unique constraint), - 'error reported by the apply worker'); +test_subscription_error( + $node_subscriber, 'test_tab1', 'INSERT', $xid, + 1, # check apply worker error + qq(duplicate key value violates unique constraint), + 'error reported by the apply worker'); # Check the table sync worker's error in the view. -test_subscription_error($node_subscriber, 'test_tab2', '', '', - 0, # check tablesync worker error - qq(duplicate key value violates unique constraint), - 'the error reported by the table sync worker'); +test_subscription_error( + $node_subscriber, 'test_tab2', '', '', + 0, # check tablesync worker error + qq(duplicate key value violates unique constraint), + 'the error reported by the table sync worker'); # Test for resetting subscription worker statistics. # Truncate test_tab1 and test_tab2 so that applying changes and table sync can # continue, respectively. -$node_subscriber->safe_psql( - 'postgres', - "TRUNCATE test_tab1, test_tab2;"); +$node_subscriber->safe_psql('postgres', "TRUNCATE test_tab1, test_tab2;"); # Wait for the data to be replicated. -$node_subscriber->poll_query_until( - 'postgres', - "SELECT count(1) > 0 FROM test_tab1"); -$node_subscriber->poll_query_until( - 'postgres', - "SELECT count(1) > 0 FROM test_tab2"); +$node_subscriber->poll_query_until('postgres', + "SELECT count(1) > 0 FROM test_tab1"); +$node_subscriber->poll_query_until('postgres', + "SELECT count(1) > 0 FROM test_tab2"); # There shouldn't be any errors in the view after dropping the subscription. -$node_subscriber->safe_psql( - 'postgres', - "DROP SUBSCRIPTION tap_sub;"); -$result = $node_subscriber->safe_psql( - 'postgres', - "SELECT count(1) FROM pg_stat_subscription_workers"); +$node_subscriber->safe_psql('postgres', "DROP SUBSCRIPTION tap_sub;"); +$result = $node_subscriber->safe_psql('postgres', + "SELECT count(1) FROM pg_stat_subscription_workers"); is($result, q(0), 'no error after dropping subscription'); $node_subscriber->stop('fast'); -- 1.8.3.1 ^ permalink raw reply [nested|flat] 113+ messages in thread
* Re: Failed transaction statistics to measure the logical replication progress @ 2021-12-21 08:59 Greg Nancarrow <[email protected]> parent: [email protected] <[email protected]> 0 siblings, 1 reply; 113+ messages in thread From: Greg Nancarrow @ 2021-12-21 08:59 UTC (permalink / raw) To: [email protected] <[email protected]>; +Cc: Kyotaro Horiguchi <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]> On Mon, Dec 20, 2021 at 8:40 PM [email protected] <[email protected]> wrote: > > Updated the patch to address your concern. > Some review comments on the v18 patches: v18-0002 doc/src/sgml/monitoring.sgml (1) tablesync worker stats? Shouldn't the comment below only mention the apply worker? (since we're no longer recording stats of the tablesync worker) + Number of transactions that failed to be applied by the table + sync worker or main apply worker in this subscription. This + counter is updated after confirming the error is not same as + the previous one. + </para></entry> Also, it should say "... the error is not the same as the previous one." src/backend/catalog/system_views.sql (2) pgstat_report_subworker_xact_end() Fix typo and some wording: BEFORE: + * This should be called before the call of process_syning_tables() not to AFTER: + * This should be called before the call of process_syncing_tables(), so to not src/backend/postmaster/pgstat.c (3) pgstat_send_subworker_xact_stats() BEFORE: + * Send a subworker transaction stats to the collector. AFTER: + * Send a subworker's transaction stats to the collector. (4) Wouldn't it be best for: + if (!TimestampDifferenceExceeds(last_report, now, PGSTAT_STAT_INTERVAL)) to be: + if (last_report != 0 && !TimestampDifferenceExceeds(last_report, now, PGSTAT_STAT_INTERVAL)) ? (5) pgstat_send_subworker_xact_stats() I think that the comment: + * Clear out the statistics buffer, so it can be re-used. should instead say: + * Clear out the supplied statistics. because the current comment infers that repWorker is pointed at the MyLogicalRepWorker buffer (which it might be but it shouldn't be relying on that) Also, I think that the function header should mention something like: "The supplied repWorker statistics are cleared upon return, to assist re-use by the caller." Regards, Greg Nancarrow Fujitsu Australia ^ permalink raw reply [nested|flat] 113+ messages in thread
* RE: Failed transaction statistics to measure the logical replication progress @ 2021-12-22 10:14 [email protected] <[email protected]> parent: Greg Nancarrow <[email protected]> 0 siblings, 2 replies; 113+ messages in thread From: [email protected] @ 2021-12-22 10:14 UTC (permalink / raw) To: 'Greg Nancarrow' <[email protected]>; +Cc: Kyotaro Horiguchi <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]> On Tuesday, December 21, 2021 6:00 PM Greg Nancarrow <[email protected]> wrote: > Some review comments on the v18 patches: Thank you for your review ! > v18-0002 > > doc/src/sgml/monitoring.sgml > (1) tablesync worker stats? > > Shouldn't the comment below only mention the apply worker? (since we're no > longer recording stats of the tablesync worker) > > + Number of transactions that failed to be applied by the table > + sync worker or main apply worker in this subscription. This > + counter is updated after confirming the error is not same as > + the previous one. > + </para></entry> > > Also, it should say "... the error is not the same as the previous one." Fixed. > src/backend/catalog/system_views.sql > (2) pgstat_report_subworker_xact_end() > > Fix typo and some wording: > > BEFORE: > + * This should be called before the call of process_syning_tables() > + not to > AFTER: > + * This should be called before the call of > process_syncing_tables(), so to not Fixed. > src/backend/postmaster/pgstat.c > (3) pgstat_send_subworker_xact_stats() > > BEFORE: > + * Send a subworker transaction stats to the collector. > AFTER: > + * Send a subworker's transaction stats to the collector. Fixed. > (4) > Wouldn't it be best for: > > + if (!TimestampDifferenceExceeds(last_report, now, > + PGSTAT_STAT_INTERVAL)) > > to be: > > + if (last_report != 0 && !TimestampDifferenceExceeds(last_report, > now, PGSTAT_STAT_INTERVAL)) > > ? I'm not sure which is better and I never have strong objection to your idea but currently I prefer the previous code because we don't need to add one extra condition (last_report != 0) in the function called really frequently and the optimization to avoid calling TimestampDifferenceExceeds works just once with your change, I'd say. We call pgstat_send_subworker_xact_stats() in the LogicalRepApplyLoop's loop. For the apply worker, this should be the first call for normal operation, before any call of the apply_dispatch(and subsequent commit-like functions which calls pgstat_send_subworker_xact_stats() in the end). In the first call, existing v18's codes (without your suggested change) just initializes 'last_report' variable because of the diff between 0 and 'now' and returns because of no stats values in commit_count and abort_count in the function. After this, 'last_report' will not be 0 for the apply worker. On the other hand, in the case I add your change, in the first call of pgstat_send_subworker_xact_stats(), similarly 'last_report' is initialized but without one call of TimestampDifferenceExceeds(), which might be the optimization effect and the function returns with no stats again. Here the 'last_report' will not be 0 after this. But, then we'll have to check the condition in the apply worker in the every loop. Besides, after the first setting of 'last_report', every call of the pgstat_send_subworker_xact_stats() calculates the time subtraction. This means the one skipped call of the function looks less worth in the case of frequent calls of the function. So, I'm not sure it' a good idea to incorporate this change. Kindly let me know if I miss something. At present, I'll keep the code as it is. > (5) pgstat_send_subworker_xact_stats() > > I think that the comment: > > + * Clear out the statistics buffer, so it can be re-used. > > should instead say: > > + * Clear out the supplied statistics. > > because the current comment infers that repWorker is pointed at the > MyLogicalRepWorker buffer (which it might be but it shouldn't be relying on > that) Also, I think that the function header should mention something like: > "The supplied repWorker statistics are cleared upon return, to assist re-use by > the caller." Fixed. Attached the new patch v19. Best Regards, Takamichi Osumi Attachments: [application/octet-stream] v19-0002-Extend-pg_stat_subscription_workers-to-include-g.patch (27.3K, ../../TYCPR01MB8373176545F7AE2CCDA23B81ED7D9@TYCPR01MB8373.jpnprd01.prod.outlook.com/2-v19-0002-Extend-pg_stat_subscription_workers-to-include-g.patch) download | inline diff: From ef80a16f4581ee2cf34108df91036cfca5fe80be Mon Sep 17 00:00:00 2001 From: Takamichi Osumi <[email protected]> Date: Wed, 22 Dec 2021 08:19:31 +0000 Subject: [PATCH v19 2/2] Extend pg_stat_subscription_workers to include general transaction statistics Categorize transactions of logical replication subscriber into three types (commit, abort, error) and introduce cumulative columns of those numbers in the pg_stat_subscription_workers. In order to avoid having a large number of entries to be created by the table synchronization for many tables, the new stats columns are utilized only by the apply worker. Author: Takamichi Osumi Reviewed-by: Amit Kapila, Masahiko Sawada, Hou Zhijie, Greg Nancarrow, Vignesh C, Ajin Cherian, Kyotaro Horiguchi Discussion: https://www.postgresql.org/message-id/OSBPR01MB48887CA8F40C8D984A6DC00CED199%40OSBPR01MB4888.jpnprd01.prod.outlook.com --- doc/src/sgml/monitoring.sgml | 41 ++++++++- src/backend/catalog/system_views.sql | 3 + src/backend/postmaster/pgstat.c | 127 ++++++++++++++++++++++++++++ src/backend/replication/logical/launcher.c | 4 + src/backend/replication/logical/worker.c | 22 +++++ src/backend/utils/adt/pgstatfuncs.c | 25 ++++-- src/include/catalog/pg_proc.dat | 6 +- src/include/pgstat.h | 32 +++++++ src/include/replication/worker_internal.h | 6 ++ src/test/regress/expected/rules.out | 5 +- src/test/subscription/t/026_worker_stats.pl | 109 +++++++++++++++++++++++- src/tools/pgindent/typedefs.list | 1 + 12 files changed, 362 insertions(+), 19 deletions(-) diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml index 62f2a33..928787c 100644 --- a/doc/src/sgml/monitoring.sgml +++ b/doc/src/sgml/monitoring.sgml @@ -629,8 +629,8 @@ postgres 27093 0.0 0.0 30096 2752 ? Ss 11:34 0:00 postgres: ser <row> <entry><structname>pg_stat_subscription_workers</structname><indexterm><primary>pg_stat_subscription_workers</primary></indexterm></entry> - <entry>One row per subscription worker, showing statistics about errors - that occurred on that subscription worker. + <entry>One row per subscription worker, showing statistics about transactions + and errors that occurred on that subscription worker. See <link linkend="monitoring-pg-stat-subscription-workers"> <structname>pg_stat_subscription_workers</structname></link> for details. </entry> @@ -3074,8 +3074,10 @@ SELECT pid, wait_event_type, wait_event FROM pg_stat_activity WHERE wait_event i The <structname>pg_stat_subscription_workers</structname> view will contain one row per subscription worker on which errors have occurred, for workers applying logical replication changes and workers handling the initial data - copy of the subscribed tables. The statistics entry is removed when the - corresponding subscription is dropped. + copy of the subscribed tables. The row corresponding to the apply + worker shows transaction statistics of the main apply worker on the + subscription. The statistics entry is removed when the corresponding + subscription is dropped. </para> <table id="pg-stat-subscription-workers" xreflabel="pg_stat_subscription_workers"> @@ -3123,6 +3125,37 @@ SELECT pid, wait_event_type, wait_event FROM pg_stat_activity WHERE wait_event i <row> <entry role="catalog_table_entry"><para role="column_definition"> + <structfield>commit_count</structfield> <type>bigint</type> + </para> + <para> + Number of transactions successfully applied in this subscription. + Both COMMIT and COMMIT PREPARED increment this counter. + </para></entry> + </row> + + <row> + <entry role="catalog_table_entry"><para role="column_definition"> + <structfield>abort_count</structfield> <type>bigint</type> + </para> + <para> + Number of transactions aborted in this subscription. + ROLLBACK PREPARED increments this counter. + </para></entry> + </row> + + <row> + <entry role="catalog_table_entry"><para role="column_definition"> + <structfield>error_count</structfield> <type>bigint</type> + </para> + <para> + Number of transactions that failed to be applied by the apply + worker in this subscription. This counter is updated after + confirming the error is not the same as the previous one. + </para></entry> + </row> + + <row> + <entry role="catalog_table_entry"><para role="column_definition"> <structfield>last_error_relid</structfield> <type>oid</type> </para> <para> diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index 61b515c..dda3cd7 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1267,6 +1267,9 @@ CREATE VIEW pg_stat_subscription_workers AS w.subid, s.subname, w.subrelid, + w.commit_count, + w.abort_count, + w.error_count, w.last_error_relid, w.last_error_command, w.last_error_xid, diff --git a/src/backend/postmaster/pgstat.c b/src/backend/postmaster/pgstat.c index 7264d2c..96edb73 100644 --- a/src/backend/postmaster/pgstat.c +++ b/src/backend/postmaster/pgstat.c @@ -382,6 +382,7 @@ static void pgstat_recv_replslot(PgStat_MsgReplSlot *msg, int len); static void pgstat_recv_tempfile(PgStat_MsgTempFile *msg, int len); static void pgstat_recv_subscription_purge(PgStat_MsgSubscriptionPurge *msg, int len); static void pgstat_recv_subworker_error(PgStat_MsgSubWorkerError *msg, int len); +static void pgstat_recv_subworker_xact_end(PgStat_MsgSubWorkerXactEnd *msg, int len); /* ------------------------------------------------------------ * Public functions called from postmaster follow @@ -1949,6 +1950,42 @@ pgstat_report_replslot_drop(const char *slotname) } /* ---------- + * pgstat_report_subworker_xact_end() - + * + * Tell the collector that worker transaction has successfully completed. + * This should be called before the call of process_syning_tables() so to not + * miss an increment of transaction stats in case it leads to a process exit. + * See process_syncing_tables_for_apply(). + * ---------- + */ +void +pgstat_report_subworker_xact_end(LogicalRepWorker *repWorker, + LogicalRepMsgType command, bool force) +{ + Assert(command == LOGICAL_REP_MSG_COMMIT || + command == LOGICAL_REP_MSG_STREAM_COMMIT || + command == LOGICAL_REP_MSG_COMMIT_PREPARED || + command == LOGICAL_REP_MSG_ROLLBACK_PREPARED); + + switch (command) + { + case LOGICAL_REP_MSG_COMMIT: + case LOGICAL_REP_MSG_STREAM_COMMIT: + case LOGICAL_REP_MSG_COMMIT_PREPARED: + repWorker->commit_count++; + break; + case LOGICAL_REP_MSG_ROLLBACK_PREPARED: + repWorker->abort_count++; + break; + default: + elog(ERROR, "unexpected logical message type as transaction end"); + break; + } + + pgstat_send_subworker_xact_stats(repWorker, force); +} + +/* ---------- * pgstat_report_subworker_error() - * * Tell the collector about the subscription worker error. @@ -3484,6 +3521,61 @@ pgstat_send_subscription_purge(PgStat_MsgSubscriptionPurge *msg) } /* ---------- + * pgstat_send_subworker_xact_stats() - + * + * Send a subworker's transaction stats to the collector. + * The supplied repWorker statistics are cleared upon return, to assist re-use by + * the caller. + * + * 'force' is true only when the subscription worker process exits. + * ---------- + */ +void +pgstat_send_subworker_xact_stats(LogicalRepWorker *repWorker, bool force) +{ + static TimestampTz last_report = 0; + PgStat_MsgSubWorkerXactEnd msg; + + if (!force) + { + TimestampTz now = GetCurrentTimestamp(); + + /* + * Don't send a message unless it's been at least PGSTAT_STAT_INTERVAL + * msec since we last sent one to avoid overloading the stats + * collector. + */ + if (!TimestampDifferenceExceeds(last_report, now, PGSTAT_STAT_INTERVAL)) + return; + last_report = now; + } + + /* + * This function can be called even if nothing at all has happened. In + * this case, avoid sending a completely empty message to the stats + * collector. + */ + if (repWorker->commit_count == 0 && repWorker->abort_count == 0) + return; + + /* + * Prepare and send the message. + */ + pgstat_setheader(&msg.m_hdr, PGSTAT_MTYPE_SUBWORKERXACTEND); + msg.m_databaseid = MyDatabaseId; + msg.m_subid = repWorker->subid; + msg.commit_count = repWorker->commit_count; + msg.abort_count = repWorker->abort_count; + pgstat_send(&msg, sizeof(PgStat_MsgSubWorkerXactEnd)); + + /* + * Clear out the supplied statistics. + */ + repWorker->commit_count = 0; + repWorker->abort_count = 0; +} + +/* ---------- * PgstatCollectorMain() - * * Start up the statistics collector process. This is the body of the @@ -3746,6 +3838,10 @@ PgstatCollectorMain(int argc, char *argv[]) pgstat_recv_subworker_error(&msg.msg_subworkererror, len); break; + case PGSTAT_MTYPE_SUBWORKERXACTEND: + pgstat_recv_subworker_xact_end(&msg.msg_subworkerxactend, len); + break; + default: break; } @@ -3965,6 +4061,9 @@ pgstat_get_subworker_entry(PgStat_StatDBEntry *dbentry, Oid subid, Oid subrelid, /* If not found, initialize the new one */ if (!found) { + subwentry->commit_count = 0; + subwentry->abort_count = 0; + subwentry->error_count = 0; subwentry->last_error_relid = InvalidOid; subwentry->last_error_command = 0; subwentry->last_error_xid = InvalidTransactionId; @@ -6153,6 +6252,27 @@ pgstat_recv_subscription_purge(PgStat_MsgSubscriptionPurge *msg, int len) } /* ---------- + * pgstat_recv_subworker_xact_end() - + * + * Process a SUBWORKERXACTEND message. + * ---------- + */ +static void +pgstat_recv_subworker_xact_end(PgStat_MsgSubWorkerXactEnd *msg, int len) +{ + PgStat_StatDBEntry *dbentry; + PgStat_StatSubWorkerEntry *wentry; + + dbentry = pgstat_get_db_entry(msg->m_databaseid, true); + wentry = pgstat_get_subworker_entry(dbentry, msg->m_subid, + InvalidOid, true); + Assert(wentry); + + wentry->commit_count += msg->commit_count; + wentry->abort_count += msg->abort_count; +} + +/* ---------- * pgstat_recv_subworker_error() - * * Process a SUBWORKERERROR message. @@ -6193,6 +6313,13 @@ pgstat_recv_subworker_error(PgStat_MsgSubWorkerError *msg, int len) subwentry->last_error_time = msg->m_timestamp; strlcpy(subwentry->last_error_message, msg->m_message, PGSTAT_SUBWORKERERROR_MSGLEN); + + /* + * Only if this is a new error reported by the apply worker, increment the + * counter of error. + */ + if (!OidIsValid(msg->m_subrelid)) + subwentry->error_count++; } /* ---------- diff --git a/src/backend/replication/logical/launcher.c b/src/backend/replication/logical/launcher.c index 3fb4caa..bb67c66 100644 --- a/src/backend/replication/logical/launcher.c +++ b/src/backend/replication/logical/launcher.c @@ -385,6 +385,8 @@ retry: TIMESTAMP_NOBEGIN(worker->last_recv_time); worker->reply_lsn = InvalidXLogRecPtr; TIMESTAMP_NOBEGIN(worker->reply_time); + worker->commit_count = 0; + worker->abort_count = 0; /* Before releasing lock, remember generation for future identification. */ generation = worker->generation; @@ -647,6 +649,8 @@ logicalrep_worker_onexit(int code, Datum arg) if (LogRepWorkerWalRcvConn) walrcv_disconnect(LogRepWorkerWalRcvConn); + pgstat_send_subworker_xact_stats(MyLogicalRepWorker, true); + logicalrep_worker_detach(); /* Cleanup fileset used for streaming transactions. */ diff --git a/src/backend/replication/logical/worker.c b/src/backend/replication/logical/worker.c index 2e79302..7bcf052 100644 --- a/src/backend/replication/logical/worker.c +++ b/src/backend/replication/logical/worker.c @@ -815,6 +815,10 @@ apply_handle_commit(StringInfo s) apply_handle_commit_internal(&commit_data); + /* Update stats */ + pgstat_report_subworker_xact_end(MyLogicalRepWorker, + LOGICAL_REP_MSG_COMMIT, false); + /* Process any tables that are being synchronized in parallel. */ process_syncing_tables(commit_data.end_lsn); @@ -960,6 +964,10 @@ apply_handle_commit_prepared(StringInfo s) store_flush_position(prepare_data.end_lsn); in_remote_transaction = false; + /* Update stats */ + pgstat_report_subworker_xact_end(MyLogicalRepWorker, + LOGICAL_REP_MSG_COMMIT_PREPARED, false); + /* Process any tables that are being synchronized in parallel. */ process_syncing_tables(prepare_data.end_lsn); @@ -1011,6 +1019,10 @@ apply_handle_rollback_prepared(StringInfo s) store_flush_position(rollback_data.rollback_end_lsn); in_remote_transaction = false; + /* Update stats */ + pgstat_report_subworker_xact_end(MyLogicalRepWorker, + LOGICAL_REP_MSG_ROLLBACK_PREPARED, false); + /* Process any tables that are being synchronized in parallel. */ process_syncing_tables(rollback_data.rollback_end_lsn); @@ -1435,6 +1447,10 @@ apply_handle_stream_commit(StringInfo s) /* unlink the files with serialized changes and subxact info */ stream_cleanup_files(MyLogicalRepWorker->subid, xid); + /* Update stats */ + pgstat_report_subworker_xact_end(MyLogicalRepWorker, + LOGICAL_REP_MSG_STREAM_COMMIT, false); + /* Process any tables that are being synchronized in parallel. */ process_syncing_tables(commit_data.end_lsn); @@ -2561,6 +2577,12 @@ LogicalRepApplyLoop(XLogRecPtr last_received) MemoryContextSwitchTo(ApplyMessageContext); + /* + * Process if we have pending transaction stats and reached + * PGSTAT_STAT_INTERVAL + */ + pgstat_send_subworker_xact_stats(MyLogicalRepWorker, false); + len = walrcv_receive(LogRepWorkerWalRcvConn, &buf, &fd); if (len != 0) diff --git a/src/backend/utils/adt/pgstatfuncs.c b/src/backend/utils/adt/pgstatfuncs.c index f529c15..87b5254 100644 --- a/src/backend/utils/adt/pgstatfuncs.c +++ b/src/backend/utils/adt/pgstatfuncs.c @@ -2415,7 +2415,7 @@ pg_stat_get_replication_slot(PG_FUNCTION_ARGS) Datum pg_stat_get_subscription_worker(PG_FUNCTION_ARGS) { -#define PG_STAT_GET_SUBSCRIPTION_WORKER_COLS 8 +#define PG_STAT_GET_SUBSCRIPTION_WORKER_COLS 11 Oid subid = PG_GETARG_OID(0); Oid subrelid; TupleDesc tupdesc; @@ -2442,17 +2442,23 @@ pg_stat_get_subscription_worker(PG_FUNCTION_ARGS) OIDOID, -1, 0); TupleDescInitEntry(tupdesc, (AttrNumber) 2, "subrelid", OIDOID, -1, 0); - TupleDescInitEntry(tupdesc, (AttrNumber) 3, "last_error_relid", + TupleDescInitEntry(tupdesc, (AttrNumber) 3, "commit_count", + INT8OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) 4, "abort_count", + INT8OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) 5, "error_count", + INT8OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) 6, "last_error_relid", OIDOID, -1, 0); - TupleDescInitEntry(tupdesc, (AttrNumber) 4, "last_error_command", + TupleDescInitEntry(tupdesc, (AttrNumber) 7, "last_error_command", TEXTOID, -1, 0); - TupleDescInitEntry(tupdesc, (AttrNumber) 5, "last_error_xid", + TupleDescInitEntry(tupdesc, (AttrNumber) 8, "last_error_xid", XIDOID, -1, 0); - TupleDescInitEntry(tupdesc, (AttrNumber) 6, "last_error_count", + TupleDescInitEntry(tupdesc, (AttrNumber) 9, "last_error_count", INT8OID, -1, 0); - TupleDescInitEntry(tupdesc, (AttrNumber) 7, "last_error_message", + TupleDescInitEntry(tupdesc, (AttrNumber) 10, "last_error_message", TEXTOID, -1, 0); - TupleDescInitEntry(tupdesc, (AttrNumber) 8, "last_error_time", + TupleDescInitEntry(tupdesc, (AttrNumber) 11, "last_error_time", TIMESTAMPTZOID, -1, 0); BlessTupleDesc(tupdesc); @@ -2470,6 +2476,11 @@ pg_stat_get_subscription_worker(PG_FUNCTION_ARGS) else nulls[i++] = true; + /* transaction stats */ + values[i++] = Int64GetDatum(wentry->commit_count); + values[i++] = Int64GetDatum(wentry->abort_count); + values[i++] = Int64GetDatum(wentry->error_count); + /* last_error_relid */ if (OidIsValid(wentry->last_error_relid)) values[i++] = ObjectIdGetDatum(wentry->last_error_relid); diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index 4d992dc..b76f84a 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -5375,9 +5375,9 @@ proname => 'pg_stat_get_subscription_worker', prorows => '1', proisstrict => 'f', proretset => 't', provolatile => 's', proparallel => 'r', prorettype => 'record', proargtypes => 'oid oid', - proallargtypes => '{oid,oid,oid,oid,oid,text,xid,int8,text,timestamptz}', - proargmodes => '{i,i,o,o,o,o,o,o,o,o}', - proargnames => '{subid,subrelid,subid,subrelid,last_error_relid,last_error_command,last_error_xid,last_error_count,last_error_message,last_error_time}', + proallargtypes => '{oid,oid,oid,oid,int8,int8,int8,oid,text,xid,int8,text,timestamptz}', + proargmodes => '{i,i,o,o,o,o,o,o,o,o,o,o,o}', + proargnames => '{subid,subrelid,subid,subrelid,commit_count,abort_count,error_count,last_error_relid,last_error_command,last_error_xid,last_error_count,last_error_message,last_error_time}', prosrc => 'pg_stat_get_subscription_worker' }, { oid => '6118', descr => 'statistics: information about subscription', proname => 'pg_stat_get_subscription', prorows => '10', proisstrict => 'f', diff --git a/src/include/pgstat.h b/src/include/pgstat.h index 5b51b58..be55b55 100644 --- a/src/include/pgstat.h +++ b/src/include/pgstat.h @@ -15,6 +15,7 @@ #include "portability/instr_time.h" #include "postmaster/pgarch.h" /* for MAX_XFN_CHARS */ #include "replication/logicalproto.h" +#include "replication/worker_internal.h" #include "utils/backend_progress.h" /* for backward compatibility */ #include "utils/backend_status.h" /* for backward compatibility */ #include "utils/hsearch.h" @@ -86,6 +87,7 @@ typedef enum StatMsgType PGSTAT_MTYPE_DISCONNECT, PGSTAT_MTYPE_SUBSCRIPTIONPURGE, PGSTAT_MTYPE_SUBWORKERERROR, + PGSTAT_MTYPE_SUBWORKERXACTEND } StatMsgType; /* ---------- @@ -558,6 +560,23 @@ typedef struct PgStat_MsgSubscriptionPurge } PgStat_MsgSubscriptionPurge; /* ---------- + * PgStat_MsgSubscriptionXactEnd Sent by the apply worker to report transaction + * ends. + * ---------- + */ +typedef struct PgStat_MsgSubWorkerXactEnd +{ + PgStat_MsgHdr m_hdr; + + /* determine the worker entry */ + Oid m_databaseid; + Oid m_subid; + + PgStat_Counter commit_count; + PgStat_Counter abort_count; +} PgStat_MsgSubWorkerXactEnd; + +/* ---------- * PgStat_MsgSubWorkerError Sent by the apply worker or the table sync * worker to report the error occurred while * processing changes. @@ -769,6 +788,7 @@ typedef union PgStat_Msg PgStat_MsgDisconnect msg_disconnect; PgStat_MsgSubscriptionPurge msg_subscriptionpurge; PgStat_MsgSubWorkerError msg_subworkererror; + PgStat_MsgSubWorkerXactEnd msg_subworkerxactend; } PgStat_Msg; @@ -1010,6 +1030,13 @@ typedef struct PgStat_StatSubWorkerEntry PgStat_StatSubWorkerKey key; /* hash key (must be first) */ /* + * Cumulative transaction statistics of subscription worker + */ + PgStat_Counter commit_count; + PgStat_Counter abort_count; + PgStat_Counter error_count; + + /* * Subscription worker error statistics representing an error that * occurred during application of changes or the initial table * synchronization. @@ -1131,6 +1158,9 @@ extern void pgstat_report_checksum_failure(void); extern void pgstat_report_replslot(const PgStat_StatReplSlotEntry *repSlotStat); extern void pgstat_report_replslot_create(const char *slotname); extern void pgstat_report_replslot_drop(const char *slotname); +extern void pgstat_report_subworker_xact_end(LogicalRepWorker *repWorker, + LogicalRepMsgType command, + bool bforce); extern void pgstat_report_subworker_error(Oid subid, Oid subrelid, Oid relid, LogicalRepMsgType command, TransactionId xid, const char *errmsg); @@ -1217,6 +1247,8 @@ extern void pgstat_send_archiver(const char *xlog, bool failed); extern void pgstat_send_bgwriter(void); extern void pgstat_send_checkpointer(void); extern void pgstat_send_wal(bool force); +extern void pgstat_send_subworker_xact_stats(LogicalRepWorker *repWorker, + bool force); /* ---------- * Support functions for the SQL-callable functions to diff --git a/src/include/replication/worker_internal.h b/src/include/replication/worker_internal.h index 9d29849..d6e4570 100644 --- a/src/include/replication/worker_internal.h +++ b/src/include/replication/worker_internal.h @@ -66,6 +66,12 @@ typedef struct LogicalRepWorker TimestampTz last_recv_time; XLogRecPtr reply_lsn; TimestampTz reply_time; + + /* + * Transaction statistics of subscription worker + */ + int64 commit_count; + int64 abort_count; } LogicalRepWorker; /* Main memory context for apply worker. Permanent during worker lifetime. */ diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out index b58b062..2b0bfae 100644 --- a/src/test/regress/expected/rules.out +++ b/src/test/regress/expected/rules.out @@ -2097,6 +2097,9 @@ pg_stat_subscription| SELECT su.oid AS subid, pg_stat_subscription_workers| SELECT w.subid, s.subname, w.subrelid, + w.commit_count, + w.abort_count, + w.error_count, w.last_error_relid, w.last_error_command, w.last_error_xid, @@ -2110,7 +2113,7 @@ pg_stat_subscription_workers| SELECT w.subid, SELECT pg_subscription_rel.srsubid AS subid, pg_subscription_rel.srrelid AS relid FROM pg_subscription_rel) sr, - (LATERAL pg_stat_get_subscription_worker(sr.subid, sr.relid) w(subid, subrelid, last_error_relid, last_error_command, last_error_xid, last_error_count, last_error_message, last_error_time) + (LATERAL pg_stat_get_subscription_worker(sr.subid, sr.relid) w(subid, subrelid, commit_count, abort_count, error_count, last_error_relid, last_error_command, last_error_xid, last_error_count, last_error_message, last_error_time) JOIN pg_subscription s ON ((w.subid = s.oid))); pg_stat_sys_indexes| SELECT pg_stat_all_indexes.relid, pg_stat_all_indexes.indexrelid, diff --git a/src/test/subscription/t/026_worker_stats.pl b/src/test/subscription/t/026_worker_stats.pl index 8005c54..99253f4 100644 --- a/src/test/subscription/t/026_worker_stats.pl +++ b/src/test/subscription/t/026_worker_stats.pl @@ -1,7 +1,7 @@ # Copyright (c) 2021, PostgreSQL Global Development Group -# Tests for subscription error stats. +# Tests for subscription stats. use strict; use warnings; use PostgreSQL::Test::Cluster; @@ -44,9 +44,32 @@ WHERE last_error_relid = '$relname'::regclass or die "Timed out while waiting for " . $msg; } +# Test whether the update of general transaction stats satisfies the expected +# condition or not. +sub confirm_transaction_stats_update +{ + my ($node, $condition, $msg) = @_; + + # Check only the stats of the apply worker + my $sql = qq[ +SELECT count(1) = 1 +FROM pg_stat_subscription_workers +WHERE subrelid IS NULL AND $condition]; + + $node->poll_query_until('postgres', $sql) + or die "Timed out while waiting for " . $msg; +} + # Create publisher node. my $node_publisher = PostgreSQL::Test::Cluster->new('publisher'); $node_publisher->init(allows_streaming => 'logical'); +$node_publisher->append_conf( + 'postgresql.conf', qq[ +logical_decoding_work_mem = 64kB +max_prepared_transactions = 10 +max_wal_senders = 10 +wal_sender_timeout = 0 +]); $node_publisher->start; # Create subscriber node. @@ -58,6 +81,7 @@ $node_subscriber->init(allows_streaming => 'logical'); $node_subscriber->append_conf( 'postgresql.conf', qq[ +max_prepared_transactions = 10 wal_retrieve_retry_interval = 2s ]); $node_subscriber->start; @@ -98,7 +122,7 @@ is($result, qq(0), 'check no subscription error'); # Create subscription. The table sync for test_tab2 on tap_sub will enter into # infinite error loop due to violating the unique constraint. $node_subscriber->safe_psql('postgres', - "CREATE SUBSCRIPTION tap_sub CONNECTION '$publisher_connstr' PUBLICATION tap_pub;" + "CREATE SUBSCRIPTION tap_sub CONNECTION '$publisher_connstr' PUBLICATION tap_pub WITH (streaming = on, two_phase = on);" ); $node_publisher->wait_for_catchup('tap_sub'); @@ -132,6 +156,11 @@ test_subscription_error( qq(duplicate key value violates unique constraint), 'error reported by the apply worker'); +confirm_transaction_stats_update( + $node_subscriber, + 'error_count = 1', + 'the error_count increment by the apply worker'); + # Check the table sync worker's error in the view. test_subscription_error( $node_subscriber, 'test_tab2', '', '', @@ -150,11 +179,83 @@ $node_subscriber->poll_query_until('postgres', $node_subscriber->poll_query_until('postgres', "SELECT count(1) > 0 FROM test_tab2"); -# There shouldn't be any errors in the view after dropping the subscription. +# Check updation of subscription worker transaction count statistics. +# COMMIT of an insertion of single record to test_tab1 +confirm_transaction_stats_update( + $node_subscriber, + 'commit_count = 1', + 'the commit_count increment by the apply worker'); + +# Some more tests for transaction stats +# PREPARE & COMMIT PREPARED +$node_publisher->safe_psql( + 'postgres', qq[ +BEGIN; +INSERT INTO test_tab1 VALUES (2); +PREPARE TRANSACTION 'gid1' +]); +$node_publisher->safe_psql('postgres', "COMMIT PREPARED 'gid1'"); +confirm_transaction_stats_update( + $node_subscriber, + 'commit_count = 2', + 'the commit_count increment by commit prepared'); + +# STREAM COMMIT +$node_publisher->safe_psql( + 'postgres', qq[ +BEGIN; +INSERT INTO test_tab1 VALUES(generate_series(1001, 2000)); +COMMIT; +]); +confirm_transaction_stats_update( + $node_subscriber, + 'commit_count = 3', + 'the commit_count increment by stream commit'); + +# STREAM PREPARE & COMMIT PREPARED +$node_publisher->safe_psql( + 'postgres', qq[ +BEGIN; +INSERT INTO test_tab1 VALUES(generate_series(2001, 3000)); +PREPARE TRANSACTION 'gid2' +]); +$node_publisher->safe_psql('postgres', "COMMIT PREPARED 'gid2'"); +confirm_transaction_stats_update( + $node_subscriber, + 'commit_count = 4', + 'the commit_count increment by streamed commit prepared'); + +# ROLLBACK PREPARED +$node_publisher->safe_psql( + 'postgres', qq[ +BEGIN; +INSERT INTO test_tab1 VALUES (3); +PREPARE TRANSACTION 'gid3'; +]); +$node_publisher->safe_psql('postgres', "ROLLBACK PREPARED 'gid3'"); +confirm_transaction_stats_update( + $node_subscriber, + 'abort_count = 1', + 'the abort_count increment by rollback prepared'); + +# STREAM PREPARE & ROLLBACK PREPARED +$node_publisher->safe_psql( + 'postgres', qq[ +BEGIN; +INSERT INTO test_tab1 VALUES (generate_series(3001, 4000)); +PREPARE TRANSACTION 'gid4'; +]); +$node_publisher->safe_psql('postgres', "ROLLBACK PREPARED 'gid4'"); +confirm_transaction_stats_update( + $node_subscriber, + 'abort_count = 2', + 'the abort_count increment by streamed rollback prepared'); + +# There shouldn't be any records in the view after dropping the subscription. $node_subscriber->safe_psql('postgres', "DROP SUBSCRIPTION tap_sub;"); $result = $node_subscriber->safe_psql('postgres', "SELECT count(1) FROM pg_stat_subscription_workers"); -is($result, q(0), 'no error after dropping subscription'); +is($result, q(0), 'no record after dropping subscription'); $node_subscriber->stop('fast'); $node_publisher->stop('fast'); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 0c61ccb..238e9fa 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -1947,6 +1947,7 @@ PgStat_MsgResetslrucounter PgStat_MsgSLRU PgStat_MsgSubscriptionPurge PgStat_MsgSubWorkerError +PgStat_MsgSubWorkerXactEnd PgStat_MsgTabpurge PgStat_MsgTabstat PgStat_MsgTempFile -- 1.8.3.1 [application/octet-stream] v19-0001-Fix-alignments-of-TAP-tests-for-pg_stat_subscrip.patch (7.4K, ../../TYCPR01MB8373176545F7AE2CCDA23B81ED7D9@TYCPR01MB8373.jpnprd01.prod.outlook.com/3-v19-0001-Fix-alignments-of-TAP-tests-for-pg_stat_subscrip.patch) download | inline diff: From a26c40ea4032fe4e62f0c732b59d26cd6da4f1ef Mon Sep 17 00:00:00 2001 From: Takamichi Osumi <[email protected]> Date: Tue, 7 Dec 2021 07:39:58 +0000 Subject: [PATCH v19 1/2] Fix alignments of TAP tests for pg_stat_subscription_workers --- src/test/subscription/t/026_worker_stats.pl | 129 ++++++++++++++-------------- 1 file changed, 63 insertions(+), 66 deletions(-) diff --git a/src/test/subscription/t/026_worker_stats.pl b/src/test/subscription/t/026_worker_stats.pl index b662be3..8005c54 100644 --- a/src/test/subscription/t/026_worker_stats.pl +++ b/src/test/subscription/t/026_worker_stats.pl @@ -11,33 +11,37 @@ use Test::More tests => 3; # Test if the error reported on pg_stat_subscription_workers view is expected. sub test_subscription_error { - my ($node, $relname, $command, $xid, $by_apply_worker, $errmsg_prefix, $msg) - = @_; + my ($node, $relname, $command, $xid, $by_apply_worker, $errmsg_prefix, + $msg) + = @_; - my $check_sql = qq[ + my $check_sql = qq[ SELECT count(1) > 0 FROM pg_stat_subscription_workers WHERE last_error_relid = '$relname'::regclass AND starts_with(last_error_message, '$errmsg_prefix')]; - # subrelid - $check_sql .= $by_apply_worker - ? qq[ AND subrelid IS NULL] - : qq[ AND subrelid = '$relname'::regclass]; - - # last_error_command - $check_sql .= $command eq '' - ? qq[ AND last_error_command IS NULL] - : qq[ AND last_error_command = '$command']; - - # last_error_xid - $check_sql .= $xid eq '' - ? qq[ AND last_error_xid IS NULL] - : qq[ AND last_error_xid = '$xid'::xid]; - - # Wait for the particular error statistics to be reported. - $node->poll_query_until('postgres', $check_sql, -) or die "Timed out while waiting for " . $msg; + # subrelid + $check_sql .= + $by_apply_worker + ? qq[ AND subrelid IS NULL] + : qq[ AND subrelid = '$relname'::regclass]; + + # last_error_command + $check_sql .= + $command eq '' + ? qq[ AND last_error_command IS NULL] + : qq[ AND last_error_command = '$command']; + + # last_error_xid + $check_sql .= + $xid eq '' + ? qq[ AND last_error_xid IS NULL] + : qq[ AND last_error_xid = '$xid'::xid]; + + # Wait for the particular error statistics to be reported. + $node->poll_query_until('postgres', $check_sql,) + or die "Timed out while waiting for " . $msg; } # Create publisher node. @@ -51,8 +55,9 @@ $node_subscriber->init(allows_streaming => 'logical'); # The subscriber will enter an infinite error loop, so we don't want # to overflow the server log with error messages. -$node_subscriber->append_conf('postgresql.conf', - qq[ +$node_subscriber->append_conf( + 'postgresql.conf', + qq[ wal_retrieve_retry_interval = 2s ]); $node_subscriber->start; @@ -61,8 +66,8 @@ $node_subscriber->start; # create the same tables but with primary keys. Also, insert some data that # will conflict with the data replicated from publisher later. $node_publisher->safe_psql( - 'postgres', - qq[ + 'postgres', + qq[ BEGIN; CREATE TABLE test_tab1 (a int); CREATE TABLE test_tab2 (a int); @@ -71,8 +76,8 @@ INSERT INTO test_tab2 VALUES (1); COMMIT; ]); $node_subscriber->safe_psql( - 'postgres', - qq[ + 'postgres', + qq[ BEGIN; CREATE TABLE test_tab1 (a int primary key); CREATE TABLE test_tab2 (a int primary key); @@ -82,81 +87,73 @@ COMMIT; # Setup publications. my $publisher_connstr = $node_publisher->connstr . ' dbname=postgres'; -$node_publisher->safe_psql( - 'postgres', - "CREATE PUBLICATION tap_pub FOR TABLE test_tab1, test_tab2;"); +$node_publisher->safe_psql('postgres', + "CREATE PUBLICATION tap_pub FOR TABLE test_tab1, test_tab2;"); # There shouldn't be any subscription errors before starting logical replication. -my $result = $node_subscriber->safe_psql( - 'postgres', - "SELECT count(1) FROM pg_stat_subscription_workers"); +my $result = $node_subscriber->safe_psql('postgres', + "SELECT count(1) FROM pg_stat_subscription_workers"); is($result, qq(0), 'check no subscription error'); # Create subscription. The table sync for test_tab2 on tap_sub will enter into # infinite error loop due to violating the unique constraint. -$node_subscriber->safe_psql( - 'postgres', - "CREATE SUBSCRIPTION tap_sub CONNECTION '$publisher_connstr' PUBLICATION tap_pub;"); +$node_subscriber->safe_psql('postgres', + "CREATE SUBSCRIPTION tap_sub CONNECTION '$publisher_connstr' PUBLICATION tap_pub;" +); $node_publisher->wait_for_catchup('tap_sub'); # Wait for initial table sync for test_tab1 to finish. $node_subscriber->poll_query_until( - 'postgres', - qq[ + 'postgres', + qq[ SELECT count(1) = 1 FROM pg_subscription_rel WHERE srrelid = 'test_tab1'::regclass AND srsubstate in ('r', 's') ]) or die "Timed out while waiting for subscriber to synchronize data"; # Check the initial data. -$result = $node_subscriber->safe_psql( - 'postgres', - "SELECT count(a) FROM test_tab1"); +$result = + $node_subscriber->safe_psql('postgres', "SELECT count(a) FROM test_tab1"); is($result, q(1), 'check initial data are copied to subscriber'); # Insert more data to test_tab1, raising an error on the subscriber due to # violation of the unique constraint on test_tab1. my $xid = $node_publisher->safe_psql( - 'postgres', - qq[ + 'postgres', + qq[ BEGIN; INSERT INTO test_tab1 VALUES (1); SELECT pg_current_xact_id()::xid; COMMIT; ]); -test_subscription_error($node_subscriber, 'test_tab1', 'INSERT', $xid, - 1, # check apply worker error - qq(duplicate key value violates unique constraint), - 'error reported by the apply worker'); +test_subscription_error( + $node_subscriber, 'test_tab1', 'INSERT', $xid, + 1, # check apply worker error + qq(duplicate key value violates unique constraint), + 'error reported by the apply worker'); # Check the table sync worker's error in the view. -test_subscription_error($node_subscriber, 'test_tab2', '', '', - 0, # check tablesync worker error - qq(duplicate key value violates unique constraint), - 'the error reported by the table sync worker'); +test_subscription_error( + $node_subscriber, 'test_tab2', '', '', + 0, # check tablesync worker error + qq(duplicate key value violates unique constraint), + 'the error reported by the table sync worker'); # Test for resetting subscription worker statistics. # Truncate test_tab1 and test_tab2 so that applying changes and table sync can # continue, respectively. -$node_subscriber->safe_psql( - 'postgres', - "TRUNCATE test_tab1, test_tab2;"); +$node_subscriber->safe_psql('postgres', "TRUNCATE test_tab1, test_tab2;"); # Wait for the data to be replicated. -$node_subscriber->poll_query_until( - 'postgres', - "SELECT count(1) > 0 FROM test_tab1"); -$node_subscriber->poll_query_until( - 'postgres', - "SELECT count(1) > 0 FROM test_tab2"); +$node_subscriber->poll_query_until('postgres', + "SELECT count(1) > 0 FROM test_tab1"); +$node_subscriber->poll_query_until('postgres', + "SELECT count(1) > 0 FROM test_tab2"); # There shouldn't be any errors in the view after dropping the subscription. -$node_subscriber->safe_psql( - 'postgres', - "DROP SUBSCRIPTION tap_sub;"); -$result = $node_subscriber->safe_psql( - 'postgres', - "SELECT count(1) FROM pg_stat_subscription_workers"); +$node_subscriber->safe_psql('postgres', "DROP SUBSCRIPTION tap_sub;"); +$result = $node_subscriber->safe_psql('postgres', + "SELECT count(1) FROM pg_stat_subscription_workers"); is($result, q(0), 'no error after dropping subscription'); $node_subscriber->stop('fast'); -- 1.8.3.1 ^ permalink raw reply [nested|flat] 113+ messages in thread
* RE: Failed transaction statistics to measure the logical replication progress @ 2021-12-22 12:37 [email protected] <[email protected]> parent: [email protected] <[email protected]> 1 sibling, 1 reply; 113+ messages in thread From: [email protected] @ 2021-12-22 12:37 UTC (permalink / raw) To: [email protected] <[email protected]>; +Cc: Kyotaro Horiguchi <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>; 'Greg Nancarrow' <[email protected]> On Mon, Dec 22, 2021 at 6:14 PM [email protected] <[email protected]> wrote: > > Attached the new patch v19. > I have a question on the v19-0002 patch: When I tested for this patch, I found pg_stat_subscription_workers has some unexpected data. For example: [Publisher] create table replica_test1(a int, b text); create publication pub1 for table replica_test1; create table replica_test2(a int, b text); create publication pub2 for table replica_test2; [Subscriber] create table replica_test1(a int, b text); create subscription sub1 CONNECTION 'dbname=postgres' publication pub1; create table replica_test2(a int, b text); create subscription sub2 CONNECTION 'dbname=postgres' publication pub2; [Publisher] insert into replica_test1 values(1,'1'); [Subscriber] select * from pg_stat_subscription_workers; -[ RECORD 1 ]------+------ Subid | 16389 subname | sub1 subrelid | commit_count | 1 ... -[ RECORD 2 ]------+------ subid | 16395 subname | sub2 subrelid | commit_count | 1 ... I originally expected only one record for "sub1". I think the reason is apply_handle_commit() always invoke pgstat_report_subworker_xact_end(). But when we insert data to replica_test1 in publish side: [In the publish] pub1's walsender1 will send three messages((LOGICAL_REP_MSG_BEGIN, LOGICAL_REP_MSG_INSERT and LOGICAL_REP_MSG_COMMIT)) to sub1's apply worker1. pub2's walsender2 will also send two messages(LOGICAL_REP_MSG_BEGIN and LOGICAL_REP_MSG_COMMIT) to sub2's apply worker2. Because inserted table is not published by pub2. [In the subscription] sub1's apply worker1 receive LOGICAL_REP_MSG_COMMIT, so invoke pgstat_report_subworker_xact_end to increase commit_count of sub1's stats. sub2's apply worker2 receive LOGICAL_REP_MSG_COMMIT, it will do the same action to increase commit_count of sub2's stats. Do we expect these commit counts which come from empty transactions ? Regards, Wang wei ^ permalink raw reply [nested|flat] 113+ messages in thread
* RE: Failed transaction statistics to measure the logical replication progress @ 2021-12-22 14:30 [email protected] <[email protected]> parent: [email protected] <[email protected]> 0 siblings, 1 reply; 113+ messages in thread From: [email protected] @ 2021-12-22 14:30 UTC (permalink / raw) To: [email protected] <[email protected]>; +Cc: Kyotaro Horiguchi <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>; 'Greg Nancarrow' <[email protected]> On Wednesday, December 22, 2021 9:38 PM Wang, Wei/王 威 <[email protected]> wrote: > I have a question on the v19-0002 patch: > > When I tested for this patch, I found pg_stat_subscription_workers has some > unexpected data. > For example: > [Publisher] > create table replica_test1(a int, b text); create publication pub1 for table > replica_test1; create table replica_test2(a int, b text); create publication pub2 > for table replica_test2; > > [Subscriber] > create table replica_test1(a int, b text); create subscription sub1 CONNECTION > 'dbname=postgres' publication pub1; create table replica_test2(a int, b text); > create subscription sub2 CONNECTION 'dbname=postgres' publication pub2; > > [Publisher] > insert into replica_test1 values(1,'1'); > > [Subscriber] > select * from pg_stat_subscription_workers; > > -[ RECORD 1 ]------+------ > Subid | 16389 > subname | sub1 > subrelid | > commit_count | 1 > ... > -[ RECORD 2 ]------+------ > subid | 16395 > subname | sub2 > subrelid | > commit_count | 1 > ... > > I originally expected only one record for "sub1". > > I think the reason is apply_handle_commit() always invoke > pgstat_report_subworker_xact_end(). > But when we insert data to replica_test1 in publish side: > [In the publish] > pub1's walsender1 will send three messages((LOGICAL_REP_MSG_BEGIN, > LOGICAL_REP_MSG_INSERT and LOGICAL_REP_MSG_COMMIT)) > to sub1's apply worker1. > pub2's walsender2 will also send two messages(LOGICAL_REP_MSG_BEGIN > and LOGICAL_REP_MSG_COMMIT) to sub2's apply worker2. Because > inserted table is not published by pub2. > > [In the subscription] > sub1's apply worker1 receive LOGICAL_REP_MSG_COMMIT, > so invoke pgstat_report_subworker_xact_end to increase > commit_count of sub1's stats. > sub2's apply worker2 receive LOGICAL_REP_MSG_COMMIT, > it will do the same action to increase commit_count of sub2's stats. > > Do we expect these commit counts which come from empty transactions ? Hi, thank you so much for your test ! This is another issue discussed in [1] where the patch in the thread is a work in progress, I think. The point you reported will bring a lot of confusion for the users, to interpret the results of the subscription stats values, if those patches including the subscription stats patch will not get committed together (like in the same version). I've confirmed that HEAD applied with v19-* and v15-0001-Skip-empty-transactions-for-logical-replication.patch on top of v19-* showed only one record, as you expected like below, although all patches are not finished yet. postgres=# select * from pg_stat_subscription_workers; -[ RECORD 1 ]------+------ subid | 16389 subname | sub1 subrelid | commit_count | 1 abort_count | 0 error_count | 0 .... IMHO, the conclusion is we are currently in the middle of fixing the behavior. [1] - https://www.postgresql.org/message-id/CAFPTHDbVLWxpfnwMxJcXq703gLXciXHE83hwKQ_0OTCZ6oLCjg%40mail.gma... Best Regards, Takamichi Osumi ^ permalink raw reply [nested|flat] 113+ messages in thread
* RE: Failed transaction statistics to measure the logical replication progress @ 2021-12-23 09:37 [email protected] <[email protected]> parent: [email protected] <[email protected]> 0 siblings, 1 reply; 113+ messages in thread From: [email protected] @ 2021-12-23 09:37 UTC (permalink / raw) To: [email protected] <[email protected]>; +Cc: Kyotaro Horiguchi <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>; 'Greg Nancarrow' <[email protected]> On Wednesday, December 22, 2021 10:30 PM [email protected] <[email protected]> wrote: > On Wednesday, December 22, 2021 8:38 PM I wrote: > > Do we expect these commit counts which come from empty transactions ? > This is another issue discussed in [1] > where the patch in the thread is a work in progress, I think. > ...... > IMHO, the conclusion is we are currently in the middle of fixing the behavior. Thank you for telling me this. After applying v19-* and v15-0001-Skip-empty-transactions-for-logical-replication.patch, I retested v19-* patches. The result of previous case looks good to me. But the results of following cases are also similar to previous unexpected result which increases commit_count or abort_count unexpectedly. [1] (Based on environment in the previous example, set TWO_PHASE=true) [Publisher] begin; insert into replica_test1 values(1,'1'); prepare transaction 'id'; commit prepared 'id'; In subscriber side, the commit_count of two records(sub1 and sub2) is increased. [2] (Based on environment in the previous example, set STREAMING=on) [Publisher] begin; INSERT INTO replica_test1 SELECT i, md5(i::text) FROM generate_series(1, 5000) s(i); commit; In subscriber side, the commit_count of two records(sub1 and sub2) is increased. [3] (Based on environment in the previous example, set TWO_PHASE=true) [Publisher] begin; insert into replica_test1 values(1,'1'); prepare transaction 'id'; rollback prepared 'id'; In subscriber side, the abort_count of two records(sub1 and sub2) is increased. I think the problem maybe is the patch you mentioned (Skip-empty-transactions-for-logical-replication.patch) is not finished yet. Share this information here. Regards, Wang wei ^ permalink raw reply [nested|flat] 113+ messages in thread
* RE: Failed transaction statistics to measure the logical replication progress @ 2021-12-31 01:12 [email protected] <[email protected]> parent: [email protected] <[email protected]> 1 sibling, 0 replies; 113+ messages in thread From: [email protected] @ 2021-12-31 01:12 UTC (permalink / raw) To: [email protected] <[email protected]>; 'Greg Nancarrow' <[email protected]>; +Cc: Kyotaro Horiguchi <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]> On Wednesday, December 22, 2021 6:14 PM [email protected] <[email protected]> wrote: > > Attached the new patch v19. > Thanks for your patch. I think it's better if you could add this patch to the commitfest. Here are some comments: 1) + <structfield>commit_count</structfield> <type>bigint</type> + </para> + <para> + Number of transactions successfully applied in this subscription. + Both COMMIT and COMMIT PREPARED increment this counter. + </para></entry> + </row> ... I think the commands (like COMMIT, COMMIT PREPARED ...) can be surrounded with "<command> </command>", thoughts? 2) +extern void pgstat_report_subworker_xact_end(LogicalRepWorker *repWorker, + LogicalRepMsgType command, + bool bforce); Should "bforce" be "force"? 3) + * This should be called before the call of process_syning_tables() so to not "process_syning_tables()" should be "process_syncing_tables()". 4) +void +pgstat_send_subworker_xact_stats(LogicalRepWorker *repWorker, bool force) +{ + static TimestampTz last_report = 0; + PgStat_MsgSubWorkerXactEnd msg; + + if (!force) + { ... + if (!TimestampDifferenceExceeds(last_report, now, PGSTAT_STAT_INTERVAL)) + return; + last_report = now; + } + ... + if (repWorker->commit_count == 0 && repWorker->abort_count == 0) + return; ... I think it's better to check commit_count and abort_count first, then check if reach PGSTAT_STAT_INTERVAL. Otherwise if commit_count and abort_count are 0, it is possible that the value of last_report has been updated but it didn't send stats in fact. In this case, last_report is not the real time that send last message. Regards, Tang ^ permalink raw reply [nested|flat] 113+ messages in thread
* Re: Failed transaction statistics to measure the logical replication progress @ 2022-03-24 03:30 Amit Kapila <[email protected]> parent: [email protected] <[email protected]> 0 siblings, 1 reply; 113+ messages in thread From: Amit Kapila @ 2022-03-24 03:30 UTC (permalink / raw) To: [email protected] <[email protected]>; +Cc: Masahiko Sawada <[email protected]>; Peter Smith <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>; Kyotaro Horiguchi <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>; Greg Nancarrow <[email protected]> On Thu, Mar 3, 2022 at 8:58 AM [email protected] <[email protected]> wrote: > This patch introduces two new subscription statistics columns (apply_commit_count and apply_rollback_count) to the pg_stat_subscription_stats view for counting cumulative transactions commits/rollbacks for a particular subscription. Now, users can already see the total number of xacts committed/rolled back in a particular database via pg_stat_database, so this can be considered duplicate information. OTOH, some users might be interested in the stats for a subscription to know how many transactions are successfully applied during replication because the information in pg_stat_database also includes the operations that happened on the node. I am not sure if it is worth adding this additional information or how useful it will be for users. Does anyone else want to weigh in on this? If nobody else sees value in this then I feel it is better to mark this patch as Returned with feedback or Rejected. We can come back to it later if we see more demand for this. -- With Regards, Amit Kapila. ^ permalink raw reply [nested|flat] 113+ messages in thread
* Re: Failed transaction statistics to measure the logical replication progress @ 2022-03-25 05:36 Masahiko Sawada <[email protected]> parent: Amit Kapila <[email protected]> 0 siblings, 1 reply; 113+ messages in thread From: Masahiko Sawada @ 2022-03-25 05:36 UTC (permalink / raw) To: Amit Kapila <[email protected]>; +Cc: [email protected] <[email protected]>; Peter Smith <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>; Kyotaro Horiguchi <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>; Greg Nancarrow <[email protected]> On Thu, Mar 24, 2022 at 12:30 PM Amit Kapila <[email protected]> wrote: > > On Thu, Mar 3, 2022 at 8:58 AM [email protected] > <[email protected]> wrote: > > > > This patch introduces two new subscription statistics columns > (apply_commit_count and apply_rollback_count) to the > pg_stat_subscription_stats view for counting cumulative transactions > commits/rollbacks for a particular subscription. Now, users can > already see the total number of xacts committed/rolled back in a > particular database via pg_stat_database, so this can be considered > duplicate information. Right. > OTOH, some users might be interested in the > stats for a subscription to know how many transactions are > successfully applied during replication because the information in > pg_stat_database also includes the operations that happened on the > node. I'm not sure how useful this information is in practice. What can we use this information for? IIRC the original purpose of this proposed feature is to provide a way for the users to understand the size and count of the succeeded and failed transactions. At some point, the patch includes the statistics of only the counts of commits, rollbacks, and errors. If new statistics also include the size, it might be useful to achieve the original goal. But I’m concerned that adding only apply_commit_count and apply_rollback_count ends up adding the duplicate statistics with no concrete use cases. > I am not sure if it is worth adding this additional information or how > useful it will be for users. Does anyone else want to weigh in on > this? > > If nobody else sees value in this then I feel it is better to mark > this patch as Returned with feedback or Rejected. We can come back to > it later if we see more demand for this. Marking as Returned with feedback makes sense to me. Regards, -- Masahiko Sawada EDB: https://www.enterprisedb.com/ ^ permalink raw reply [nested|flat] 113+ messages in thread
* RE: Failed transaction statistics to measure the logical replication progress @ 2022-03-28 06:10 [email protected] <[email protected]> parent: Masahiko Sawada <[email protected]> 0 siblings, 0 replies; 113+ messages in thread From: [email protected] @ 2022-03-28 06:10 UTC (permalink / raw) To: 'Masahiko Sawada' <[email protected]>; Amit Kapila <[email protected]>; +Cc: Peter Smith <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>; Kyotaro Horiguchi <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>; Greg Nancarrow <[email protected]> Hi On Friday, March 25, 2022 2:36 PM Masahiko Sawada <[email protected]> wrote: > On Thu, Mar 24, 2022 at 12:30 PM Amit Kapila <[email protected]> > wrote: > > > > On Thu, Mar 3, 2022 at 8:58 AM [email protected] > > <[email protected]> wrote: > > > > > > > This patch introduces two new subscription statistics columns > > (apply_commit_count and apply_rollback_count) to the > > pg_stat_subscription_stats view for counting cumulative transactions > > commits/rollbacks for a particular subscription. Now, users can > > already see the total number of xacts committed/rolled back in a > > particular database via pg_stat_database, so this can be considered > > duplicate information. > > Right. ... > > I am not sure if it is worth adding this additional information or how > > useful it will be for users. Does anyone else want to weigh in on > > this? > > > > If nobody else sees value in this then I feel it is better to mark > > this patch as Returned with feedback or Rejected. We can come back to > > it later if we see more demand for this. > > Marking as Returned with feedback makes sense to me. OK. Thank you so much for sharing your opinions, Sawada-san and Amit-san. I changed the status of this entry to "Returned with feedback" accordingly. Best Regards, Takamichi Osumi ^ permalink raw reply [nested|flat] 113+ messages in thread
end of thread, other threads:[~2022-03-28 06:10 UTC | newest] Thread overview: 113+ messages (download: mbox mbox.gz follow: Atom feed) -- links below jump to the message on this page -- 2020-03-20 04:03 [PATCH v8 1/8] nodeAgg: separate context for each hashtable Justin Pryzby <[email protected]> 2021-07-08 06:54 Failed transaction statistics to measure the logical replication progress [email protected] <[email protected]> 2021-07-13 05:50 ` Re: Failed transaction statistics to measure the logical replication progress vignesh C <[email protected]> 2021-07-13 06:59 ` RE: Failed transaction statistics to measure the logical replication progress [email protected] <[email protected]> 2021-07-27 06:59 ` Re: Failed transaction statistics to measure the logical replication progress Ajin Cherian <[email protected]> 2021-07-28 12:56 ` RE: Failed transaction statistics to measure the logical replication progress [email protected] <[email protected]> 2021-07-29 01:49 ` Re: Failed transaction statistics to measure the logical replication progress Masahiko Sawada <[email protected]> 2021-08-02 05:52 ` RE: Failed transaction statistics to measure the logical replication progress [email protected] <[email protected]> 2021-08-02 07:42 ` Re: Failed transaction statistics to measure the logical replication progress Masahiko Sawada <[email protected]> 2021-08-03 02:47 ` Re: Failed transaction statistics to measure the logical replication progress Amit Kapila <[email protected]> 2021-08-03 05:28 ` Re: Failed transaction statistics to measure the logical replication progress Masahiko Sawada <[email protected]> 2021-08-03 06:09 ` RE: Failed transaction statistics to measure the logical replication progress [email protected] <[email protected]> 2021-08-03 09:11 ` Re: Failed transaction statistics to measure the logical replication progress Amit Kapila <[email protected]> 2021-08-04 00:48 ` Re: Failed transaction statistics to measure the logical replication progress Masahiko Sawada <[email protected]> 2021-08-04 03:17 ` Re: Failed transaction statistics to measure the logical replication progress Amit Kapila <[email protected]> 2021-08-04 07:04 ` Re: Failed transaction statistics to measure the logical replication progress Masahiko Sawada <[email protected]> 2021-08-04 12:58 ` Re: Failed transaction statistics to measure the logical replication progress Amit Kapila <[email protected]> 2021-09-02 02:22 ` RE: Failed transaction statistics to measure the logical replication progress [email protected] <[email protected]> 2021-09-09 08:03 ` RE: Failed transaction statistics to measure the logical replication progress [email protected] <[email protected]> 2021-09-14 03:11 ` RE: Failed transaction statistics to measure the logical replication progress [email protected] <[email protected]> 2021-09-22 04:39 ` RE: Failed transaction statistics to measure the logical replication progress [email protected] <[email protected]> 2021-09-27 04:42 ` Re: Failed transaction statistics to measure the logical replication progress Amit Kapila <[email protected]> 2021-09-28 01:55 ` RE: Failed transaction statistics to measure the logical replication progress [email protected] <[email protected]> 2021-09-28 04:54 ` Re: Failed transaction statistics to measure the logical replication progress Amit Kapila <[email protected]> 2021-09-28 06:04 ` Re: Failed transaction statistics to measure the logical replication progress Masahiko Sawada <[email protected]> 2021-09-28 10:04 ` Re: Failed transaction statistics to measure the logical replication progress Amit Kapila <[email protected]> 2021-09-29 05:59 ` Re: Failed transaction statistics to measure the logical replication progress Greg Nancarrow <[email protected]> 2021-09-29 06:27 ` RE: Failed transaction statistics to measure the logical replication progress [email protected] <[email protected]> 2021-09-29 06:05 ` RE: Failed transaction statistics to measure the logical replication progress [email protected] <[email protected]> 2021-09-29 10:51 ` Re: Failed transaction statistics to measure the logical replication progress Amit Kapila <[email protected]> 2021-09-30 02:23 ` RE: Failed transaction statistics to measure the logical replication progress Osumi, Takamichi/大墨 昂道 <[email protected]> 2021-09-29 11:18 ` Re: Failed transaction statistics to measure the logical replication progress Masahiko Sawada <[email protected]> 2021-09-29 11:55 ` Re: Failed transaction statistics to measure the logical replication progress Amit Kapila <[email protected]> 2021-09-30 02:47 ` Re: Failed transaction statistics to measure the logical replication progress Masahiko Sawada <[email protected]> 2021-09-30 03:12 ` Re: Failed transaction statistics to measure the logical replication progress Amit Kapila <[email protected]> 2021-09-30 02:52 ` RE: Failed transaction statistics to measure the logical replication progress Hou, Zhijie/侯 志杰 <[email protected]> 2021-09-30 04:06 ` RE: Failed transaction statistics to measure the logical replication progress Osumi, Takamichi/大墨 昂道 <[email protected]> 2021-09-30 04:14 ` Re: Failed transaction statistics to measure the logical replication progress Amit Kapila <[email protected]> 2021-09-30 07:32 ` RE: Failed transaction statistics to measure the logical replication progress Osumi, Takamichi/大墨 昂道 <[email protected]> 2021-09-30 11:13 ` Re: Failed transaction statistics to measure the logical replication progress Amit Kapila <[email protected]> 2021-10-01 14:52 ` RE: Failed transaction statistics to measure the logical replication progress [email protected] <[email protected]> 2021-10-14 03:53 ` RE: Failed transaction statistics to measure the logical replication progress [email protected] <[email protected]> 2021-10-14 06:13 ` RE: Failed transaction statistics to measure the logical replication progress [email protected] <[email protected]> 2021-10-18 02:51 ` RE: Failed transaction statistics to measure the logical replication progress [email protected] <[email protected]> 2021-10-19 02:51 ` RE: Failed transaction statistics to measure the logical replication progress [email protected] <[email protected]> 2021-10-18 10:03 ` Re: Failed transaction statistics to measure the logical replication progress Amit Kapila <[email protected]> 2021-10-19 01:41 ` RE: Failed transaction statistics to measure the logical replication progress [email protected] <[email protected]> 2021-10-21 01:18 ` Re: Failed transaction statistics to measure the logical replication progress Masahiko Sawada <[email protected]> 2021-10-21 08:34 ` Re: Failed transaction statistics to measure the logical replication progress Amit Kapila <[email protected]> 2021-10-28 14:18 ` RE: Failed transaction statistics to measure the logical replication progress [email protected] <[email protected]> 2021-11-01 13:18 ` RE: Failed transaction statistics to measure the logical replication progress [email protected] <[email protected]> 2021-11-02 12:07 ` RE: Failed transaction statistics to measure the logical replication progress [email protected] <[email protected]> 2021-11-04 00:53 ` Re: Failed transaction statistics to measure the logical replication progress Greg Nancarrow <[email protected]> 2021-11-05 08:11 ` RE: Failed transaction statistics to measure the logical replication progress [email protected] <[email protected]> 2021-11-08 06:12 ` Re: Failed transaction statistics to measure the logical replication progress vignesh C <[email protected]> 2021-11-09 11:39 ` RE: Failed transaction statistics to measure the logical replication progress [email protected] <[email protected]> 2021-11-08 23:50 ` Re: Failed transaction statistics to measure the logical replication progress Greg Nancarrow <[email protected]> 2021-11-09 03:07 ` Re: Failed transaction statistics to measure the logical replication progress Greg Nancarrow <[email protected]> 2021-11-09 11:35 ` RE: Failed transaction statistics to measure the logical replication progress [email protected] <[email protected]> 2021-11-09 11:55 ` RE: Failed transaction statistics to measure the logical replication progress [email protected] <[email protected]> 2021-11-10 06:43 ` Re: Failed transaction statistics to measure the logical replication progress Dilip Kumar <[email protected]> 2021-11-10 09:12 ` RE: Failed transaction statistics to measure the logical replication progress [email protected] <[email protected]> 2021-11-15 09:32 ` RE: Failed transaction statistics to measure the logical replication progress [email protected] <[email protected]> 2021-11-10 10:13 ` Re: Failed transaction statistics to measure the logical replication progress vignesh C <[email protected]> 2021-11-11 12:47 ` Re: Failed transaction statistics to measure the logical replication progress vignesh C <[email protected]> 2021-11-15 09:27 ` RE: Failed transaction statistics to measure the logical replication progress [email protected] <[email protected]> 2021-11-15 12:13 ` RE: Failed transaction statistics to measure the logical replication progress [email protected] <[email protected]> 2021-11-16 12:34 ` RE: Failed transaction statistics to measure the logical replication progress [email protected] <[email protected]> 2021-11-17 03:18 ` Re: Failed transaction statistics to measure the logical replication progress Masahiko Sawada <[email protected]> 2021-11-17 04:14 ` RE: Failed transaction statistics to measure the logical replication progress [email protected] <[email protected]> 2021-11-17 13:00 ` Re: Failed transaction statistics to measure the logical replication progress Amit Kapila <[email protected]> 2021-11-17 13:42 ` RE: Failed transaction statistics to measure the logical replication progress [email protected] <[email protected]> 2021-11-18 03:26 ` Re: Failed transaction statistics to measure the logical replication progress Amit Kapila <[email protected]> 2021-11-18 14:44 ` RE: Failed transaction statistics to measure the logical replication progress [email protected] <[email protected]> 2021-11-19 14:10 ` Re: Failed transaction statistics to measure the logical replication progress Masahiko Sawada <[email protected]> 2021-11-20 06:25 ` Re: Failed transaction statistics to measure the logical replication progress Amit Kapila <[email protected]> 2021-11-24 00:19 ` Re: Failed transaction statistics to measure the logical replication progress Masahiko Sawada <[email protected]> 2021-11-22 04:28 ` Re: Failed transaction statistics to measure the logical replication progress vignesh C <[email protected]> 2021-11-23 06:21 ` Re: Failed transaction statistics to measure the logical replication progress Greg Nancarrow <[email protected]> 2021-11-24 00:26 ` Re: Failed transaction statistics to measure the logical replication progress Masahiko Sawada <[email protected]> 2021-12-01 09:34 ` RE: Failed transaction statistics to measure the logical replication progress [email protected] <[email protected]> 2021-12-03 06:11 ` Re: Failed transaction statistics to measure the logical replication progress vignesh C <[email protected]> 2021-12-04 13:01 ` RE: Failed transaction statistics to measure the logical replication progress [email protected] <[email protected]> 2021-12-06 14:27 ` Re: Failed transaction statistics to measure the logical replication progress vignesh C <[email protected]> 2021-12-07 09:42 ` RE: Failed transaction statistics to measure the logical replication progress [email protected] <[email protected]> 2021-12-13 09:19 ` Re: Failed transaction statistics to measure the logical replication progress Amit Kapila <[email protected]> 2021-12-13 12:18 ` RE: Failed transaction statistics to measure the logical replication progress [email protected] <[email protected]> 2021-12-14 02:28 ` Re: Failed transaction statistics to measure the logical replication progress Amit Kapila <[email protected]> 2021-12-14 13:28 ` RE: Failed transaction statistics to measure the logical replication progress [email protected] <[email protected]> 2021-12-15 05:09 ` Re: Failed transaction statistics to measure the logical replication progress Masahiko Sawada <[email protected]> 2021-12-15 12:51 ` Re: Failed transaction statistics to measure the logical replication progress vignesh C <[email protected]> 2021-12-16 06:59 ` RE: Failed transaction statistics to measure the logical replication progress [email protected] <[email protected]> 2021-12-16 11:36 ` RE: Failed transaction statistics to measure the logical replication progress [email protected] <[email protected]> 2021-12-16 13:21 ` RE: Failed transaction statistics to measure the logical replication progress [email protected] <[email protected]> 2021-12-17 05:03 ` Re: Failed transaction statistics to measure the logical replication progress Kyotaro Horiguchi <[email protected]> 2021-12-20 09:40 ` RE: Failed transaction statistics to measure the logical replication progress [email protected] <[email protected]> 2021-12-21 08:59 ` Re: Failed transaction statistics to measure the logical replication progress Greg Nancarrow <[email protected]> 2021-12-22 10:14 ` RE: Failed transaction statistics to measure the logical replication progress [email protected] <[email protected]> 2021-12-22 12:37 ` RE: Failed transaction statistics to measure the logical replication progress [email protected] <[email protected]> 2021-12-22 14:30 ` RE: Failed transaction statistics to measure the logical replication progress [email protected] <[email protected]> 2021-12-23 09:37 ` RE: Failed transaction statistics to measure the logical replication progress [email protected] <[email protected]> 2022-03-24 03:30 ` Re: Failed transaction statistics to measure the logical replication progress Amit Kapila <[email protected]> 2022-03-25 05:36 ` Re: Failed transaction statistics to measure the logical replication progress Masahiko Sawada <[email protected]> 2022-03-28 06:10 ` RE: Failed transaction statistics to measure the logical replication progress [email protected] <[email protected]> 2021-12-31 01:12 ` RE: Failed transaction statistics to measure the logical replication progress [email protected] <[email protected]> 2021-12-16 07:08 ` Re: Failed transaction statistics to measure the logical replication progress Greg Nancarrow <[email protected]> 2021-12-16 11:39 ` RE: Failed transaction statistics to measure the logical replication progress [email protected] <[email protected]> 2021-12-13 15:45 ` Re: Failed transaction statistics to measure the logical replication progress vignesh C <[email protected]> 2021-12-16 11:41 ` RE: Failed transaction statistics to measure the logical replication progress [email protected] <[email protected]> 2021-11-18 11:34 ` Re: Failed transaction statistics to measure the logical replication progress vignesh C <[email protected]> 2021-11-18 14:39 ` RE: Failed transaction statistics to measure the logical replication progress [email protected] <[email protected]> 2021-11-15 09:31 ` RE: Failed transaction statistics to measure the logical replication progress [email protected] <[email protected]> 2021-08-03 05:29 ` Re: Failed transaction statistics to measure the logical replication progress vignesh C <[email protected]>
This inbox is served by agora; see mirroring instructions for how to clone and mirror all data and code used for this inbox