public inbox for [email protected]help / color / mirror / Atom feed
[PATCH 2/6] Add monitoring aid for max_slot_wal_keep_size 9+ messages / 5 participants [nested] [flat]
* [PATCH 2/6] Add monitoring aid for max_slot_wal_keep_size @ 2017-12-21 12:23 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 9+ messages in thread From: Kyotaro Horiguchi @ 2017-12-21 12:23 UTC (permalink / raw) Adds two columns "status" and "remain" in pg_replication_slot. Setting max_slot_wal_keep_size, replication connections may lose sync by a long delay. The "status" column shows whether the slot is reconnectable or not, or about to lose reserving WAL segments. The "remain" column shows the remaining bytes of WAL that can be advance until the slot loses required WAL records. --- contrib/test_decoding/expected/ddl.out | 4 +- contrib/test_decoding/sql/ddl.sql | 2 + src/backend/access/transam/xlog.c | 152 +++++++++++++++++++++++++++++++-- src/backend/catalog/system_views.sql | 4 +- src/backend/replication/slotfuncs.c | 16 +++- src/include/access/xlog.h | 1 + src/include/catalog/pg_proc.dat | 6 +- src/test/regress/expected/rules.out | 6 +- 8 files changed, 174 insertions(+), 17 deletions(-) diff --git a/contrib/test_decoding/expected/ddl.out b/contrib/test_decoding/expected/ddl.out index 2bd28e6d15..9f42dc0991 100644 --- a/contrib/test_decoding/expected/ddl.out +++ b/contrib/test_decoding/expected/ddl.out @@ -723,8 +723,8 @@ SELECT pg_drop_replication_slot('regression_slot'); (1 row) /* check that the slot is gone */ +\x SELECT * FROM pg_replication_slots; - slot_name | plugin | slot_type | datoid | database | temporary | active | active_pid | xmin | catalog_xmin | restart_lsn | confirmed_flush_lsn ------------+--------+-----------+--------+----------+-----------+--------+------------+------+--------------+-------------+--------------------- (0 rows) +\x diff --git a/contrib/test_decoding/sql/ddl.sql b/contrib/test_decoding/sql/ddl.sql index a55086443c..e793ddd366 100644 --- a/contrib/test_decoding/sql/ddl.sql +++ b/contrib/test_decoding/sql/ddl.sql @@ -387,4 +387,6 @@ SELECT data FROM pg_logical_slot_get_changes('regression_slot', NULL, NULL, 'inc SELECT pg_drop_replication_slot('regression_slot'); /* check that the slot is gone */ +\x SELECT * FROM pg_replication_slots; +\x diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index 998b779277..9623469a5e 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -873,7 +873,9 @@ static void checkTimeLineSwitch(XLogRecPtr lsn, TimeLineID newTLI, static void LocalSetXLogInsertAllowed(void); static void CreateEndOfRecoveryRecord(void); static void CheckPointGuts(XLogRecPtr checkPointRedo, int flags); -static XLogSegNo GetOldestKeepSegment(XLogRecPtr currpos, XLogRecPtr minSlotPtr); +static XLogSegNo GetOldestXLogFileSegNo(void); +static XLogSegNo GetOldestKeepSegment(XLogRecPtr currpos, XLogRecPtr minSlotPtr, + XLogRecPtr targetLSN, uint64 *restBytes); static void KeepLogSeg(XLogRecPtr recptr, XLogSegNo *logSegNo); static XLogRecPtr XLogGetReplicationSlotMinimumLSN(void); @@ -6664,6 +6666,12 @@ StartupXLOG(void) */ StartupReplicationOrigin(); + /* + * Initialize lastRemovedSegNo looking pg_wal directory. The minimum + * segment number is 1 so wrap-around cannot happen. + */ + XLogCtl->lastRemovedSegNo = GetOldestXLogFileSegNo() - 1; + /* * Initialize unlogged LSN. On a clean shutdown, it's restored from the * control file. On recovery, all unlogged relations are blown away, so @@ -9331,6 +9339,96 @@ CreateRestartPoint(int flags) return true; } + +/* + * Finds the oldest segment number in XLOG directory. + * + * This function is intended to be used to initialize + * XLogCtl->lastRemovedSegNo. + */ +static XLogSegNo +GetOldestXLogFileSegNo(void) +{ + DIR *xldir; + struct dirent *xlde; + XLogSegNo segno = 0; + + xldir = AllocateDir(XLOGDIR); + while ((xlde = ReadDir(xldir, XLOGDIR)) != NULL) + { + TimeLineID tli; + XLogSegNo fsegno; + + /* Ignore files that are not XLOG segments */ + if (!IsXLogFileName(xlde->d_name) && + !IsPartialXLogFileName(xlde->d_name)) + continue; + + XLogFromFileName(xlde->d_name, &tli, &fsegno, wal_segment_size); + + /* + * Get minimum segment ignoring timeline ID, the same way with + * RemoveOldXlogFiles(). + */ + if (segno == 0 || fsegno < segno) + segno = fsegno; + } + + FreeDir(xldir); + + return segno; +} + +/* + * Returns availability of the record at given targetLSN. + * + * Returns three kinds of value in string. + * "streaming" means the WAL record at targetLSN is available. + * "keeping" means it is still available but about to be removed at the next + * checkpoint. + * "lost" means it is already removed. + * + * If restBytes is not NULL, sets the remaining LSN bytes until the segment + * for targetLSN will be removed. + */ +char * +GetLsnAvailability(XLogRecPtr targetLSN, uint64 *restBytes) +{ + XLogRecPtr currpos; + XLogRecPtr slotPtr; + XLogSegNo targetSeg; + XLogSegNo tailSeg; + XLogSegNo oldestSeg; + + Assert(!XLogRecPtrIsInvalid(targetLSN)); + Assert(restBytes); + + currpos = GetXLogWriteRecPtr(); + + SpinLockAcquire(&XLogCtl->info_lck); + oldestSeg = XLogCtl->lastRemovedSegNo; + SpinLockRelease(&XLogCtl->info_lck); + + /* oldest segment is just after the last removed segment */ + oldestSeg++; + + XLByteToSeg(targetLSN, targetSeg, wal_segment_size); + + slotPtr = XLogGetReplicationSlotMinimumLSN(); + tailSeg = GetOldestKeepSegment(currpos, slotPtr, targetLSN, restBytes); + + /* targetSeg is being reserved by slots */ + if (tailSeg <= targetSeg) + return "streaming"; + + /* targetSeg is no longer reserved but still available */ + if (oldestSeg <= targetSeg) + return "keeping"; + + /* targetSeg has gone */ + return "lost"; +} + /* * Returns minimum segment number that the next checkpoint must leave * considering wal_keep_segments, replication slots and @@ -9338,13 +9436,19 @@ CreateRestartPoint(int flags) * * currLSN is the current insert location. * minSlotLSN is the minimum restart_lsn of all active slots. + * targetLSN is used when restBytes is not NULL. + * + * If restBytes is not NULL, sets the remaining LSN bytes until the segment + * for targetLSN will be removed. */ static XLogSegNo -GetOldestKeepSegment(XLogRecPtr currLSN, XLogRecPtr minSlotLSN) +GetOldestKeepSegment(XLogRecPtr currLSN, XLogRecPtr minSlotLSN, + XLogRecPtr targetLSN, uint64 *restBytes) { XLogSegNo currSeg; XLogSegNo minSlotSeg; uint64 keepSegs = 0; /* # of segments actually kept */ + uint64 limitSegs = 0; /* # of maximum segments possibly kept */ XLByteToSeg(currLSN, currSeg, wal_segment_size); XLByteToSeg(minSlotLSN, minSlotSeg, wal_segment_size); @@ -9359,8 +9463,6 @@ GetOldestKeepSegment(XLogRecPtr currLSN, XLogRecPtr minSlotLSN) /* Cap keepSegs by max_slot_wal_keep_size */ if (max_slot_wal_keep_size_mb >= 0) { - uint64 limitSegs; - limitSegs = ConvertToXSegs(max_slot_wal_keep_size_mb, wal_segment_size); /* Reduce it if slots already reserves too many. */ @@ -9368,9 +9470,42 @@ GetOldestKeepSegment(XLogRecPtr currLSN, XLogRecPtr minSlotLSN) keepSegs = limitSegs; } - /* but, keep at least wal_keep_segments segments if any */ - if (wal_keep_segments > 0 && keepSegs < wal_keep_segments) - keepSegs = wal_keep_segments; + if (wal_keep_segments > 0) + { + /* but, keep at least wal_keep_segments segments if any */ + if (keepSegs < wal_keep_segments) + keepSegs = wal_keep_segments; + + /* ditto for limitSegs */ + if (limitSegs < wal_keep_segments) + limitSegs = wal_keep_segments; + } + + /* + * If requested, calculate the remaining LSN bytes until the slot gives up + * keeping WAL records. + */ + if (restBytes) + { + uint64 fragbytes; + XLogSegNo targetSeg; + + *restBytes = 0; + + XLByteToSeg(targetLSN, targetSeg, wal_segment_size); + + /* avoid underflow */ + if (max_slot_wal_keep_size_mb >= 0 && currSeg <= targetSeg + limitSegs) + { + /* + * This slot still has all required segments. Calculate how many + * LSN bytes the slot has until it loses targetLSN. + */ + fragbytes = wal_segment_size - (currLSN % wal_segment_size); + XLogSegNoOffsetToRecPtr(targetSeg + limitSegs - currSeg, fragbytes, + wal_segment_size, *restBytes); + } + } /* avoid underflow, don't go below 1 */ if (currSeg <= keepSegs) @@ -9400,7 +9535,8 @@ KeepLogSeg(XLogRecPtr recptr, XLogSegNo *logSegNo) /* * We should keep certain number of WAL segments after this checkpoint. */ - minSegNo = GetOldestKeepSegment(recptr, slotminptr); + minSegNo = + GetOldestKeepSegment(recptr, slotminptr, InvalidXLogRecPtr, NULL); /* * Warn the checkpoint is going to flush the segments required by diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index 3e229c693c..26a6c3bfd5 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -800,7 +800,9 @@ CREATE VIEW pg_replication_slots AS L.xmin, L.catalog_xmin, L.restart_lsn, - L.confirmed_flush_lsn + L.confirmed_flush_lsn, + L.wal_status, + L.remain FROM pg_get_replication_slots() AS L LEFT JOIN pg_database D ON (L.datoid = D.oid); diff --git a/src/backend/replication/slotfuncs.c b/src/backend/replication/slotfuncs.c index 224dd920c8..cac66978ed 100644 --- a/src/backend/replication/slotfuncs.c +++ b/src/backend/replication/slotfuncs.c @@ -185,7 +185,7 @@ pg_drop_replication_slot(PG_FUNCTION_ARGS) Datum pg_get_replication_slots(PG_FUNCTION_ARGS) { -#define PG_GET_REPLICATION_SLOTS_COLS 11 +#define PG_GET_REPLICATION_SLOTS_COLS 13 ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; TupleDesc tupdesc; Tuplestorestate *tupstore; @@ -307,6 +307,20 @@ pg_get_replication_slots(PG_FUNCTION_ARGS) else nulls[i++] = true; + if (restart_lsn == InvalidXLogRecPtr) + { + values[i++] = CStringGetTextDatum("unknown"); + values[i++] = LSNGetDatum(InvalidXLogRecPtr); + } + else + { + uint64 remaining_bytes; + + values[i++] = CStringGetTextDatum( + GetLsnAvailability(restart_lsn, &remaining_bytes)); + values[i++] = Int64GetDatum(remaining_bytes); + } + tuplestore_putvalues(tupstore, tupdesc, values, nulls); } LWLockRelease(ReplicationSlotControlLock); diff --git a/src/include/access/xlog.h b/src/include/access/xlog.h index b2eb30b779..b0cdba6d7a 100644 --- a/src/include/access/xlog.h +++ b/src/include/access/xlog.h @@ -302,6 +302,7 @@ extern void ShutdownXLOG(int code, Datum arg); extern void InitXLOGAccess(void); extern void CreateCheckPoint(int flags); extern bool CreateRestartPoint(int flags); +extern char *GetLsnAvailability(XLogRecPtr targetLSN, uint64 *restBytes); extern void XLogPutNextOid(Oid nextOid); extern XLogRecPtr XLogRestorePoint(const char *rpName); extern void UpdateFullPageWrites(void); diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index a4e173b484..0014bd18ee 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -9685,9 +9685,9 @@ proname => 'pg_get_replication_slots', prorows => '10', proisstrict => 'f', proretset => 't', provolatile => 's', prorettype => 'record', proargtypes => '', - proallargtypes => '{name,name,text,oid,bool,bool,int4,xid,xid,pg_lsn,pg_lsn}', - proargmodes => '{o,o,o,o,o,o,o,o,o,o,o}', - proargnames => '{slot_name,plugin,slot_type,datoid,temporary,active,active_pid,xmin,catalog_xmin,restart_lsn,confirmed_flush_lsn}', + proallargtypes => '{name,name,text,oid,bool,bool,int4,xid,xid,pg_lsn,pg_lsn,text,int8}', + proargmodes => '{o,o,o,o,o,o,o,o,o,o,o,o,o}', + proargnames => '{slot_name,plugin,slot_type,datoid,temporary,active,active_pid,xmin,catalog_xmin,restart_lsn,confirmed_flush_lsn,wal_status,remain}', prosrc => 'pg_get_replication_slots' }, { oid => '3786', descr => 'set up a logical replication slot', proname => 'pg_create_logical_replication_slot', provolatile => 'v', diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out index 98f417cb57..d0b05a26d5 100644 --- a/src/test/regress/expected/rules.out +++ b/src/test/regress/expected/rules.out @@ -1459,8 +1459,10 @@ pg_replication_slots| SELECT l.slot_name, l.xmin, l.catalog_xmin, l.restart_lsn, - l.confirmed_flush_lsn - FROM (pg_get_replication_slots() l(slot_name, plugin, slot_type, datoid, temporary, active, active_pid, xmin, catalog_xmin, restart_lsn, confirmed_flush_lsn) + l.confirmed_flush_lsn, + l.wal_status, + l.remain + FROM (pg_get_replication_slots() l(slot_name, plugin, slot_type, datoid, temporary, active, active_pid, xmin, catalog_xmin, restart_lsn, confirmed_flush_lsn, wal_status, remain) LEFT JOIN pg_database d ON ((l.datoid = d.oid))); pg_roles| SELECT pg_authid.rolname, pg_authid.rolsuper, -- 2.16.3 ----Next_Part(Fri_Feb_22_14_12_28_2019_076)-- Content-Type: Text/X-Patch; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="v13-0003-Add-primary_slot_name-to-init_from_backup-in-TAP-tes.patch" ^ permalink raw reply [nested|flat] 9+ messages in thread
* Re: pg_walinspect - a new extension to get raw WAL data and WAL stats @ 2022-04-04 03:45 Bharath Rupireddy <[email protected]> 2022-04-06 05:02 ` Re: pg_walinspect - a new extension to get raw WAL data and WAL stats Jeff Davis <[email protected]> 0 siblings, 1 reply; 9+ messages in thread From: Bharath Rupireddy @ 2022-04-04 03:45 UTC (permalink / raw) To: Jeff Davis <[email protected]>; +Cc: RKN Sai Krishna <[email protected]>; Andres Freund <[email protected]>; Kyotaro Horiguchi <[email protected]>; Ashutosh Sharma <[email protected]>; Stephen Frost <[email protected]>; Robert Haas <[email protected]>; Andrew Dunstan <[email protected]>; Greg Stark <[email protected]>; Jeremy Schneider <[email protected]>; Bruce Momjian <[email protected]>; PostgreSQL Hackers <[email protected]>; SATYANARAYANA NARLAPURAM <[email protected]>; [email protected]; [email protected] On Sat, Apr 2, 2022 at 5:05 AM Jeff Davis <[email protected]> wrote: > > On Sat, 2022-03-26 at 10:31 +0530, Bharath Rupireddy wrote: > > Attaching v16 patch-set, only change in v16-0002-pg_walinspect.patch, > > others remain the same. > > I looked more closely at this patch. Thanks Jeff for reviewing this. > * It seems that pg_get_wal_record() is not returning the correct raw > data for the record. I tested with pg_logical_emit_message, and the > message isn't there. pg_get_wal_record_info() uses XLogRecordGetData(), > which seems closer to what I expect. > > * I'm a little unclear on the purpose of pg_get_wal_record(). What does > it offer that the other functions don't? My intention is to return the overall undecoded WAL record [5] i.e. the data starting from XLogReadRecord's output [6] till length XLogRecGetTotalLen(xlogreader);. Please see [7], where Andres agreed to have this function, I also mentioned a possible use-case there. pg_get_wal_record_info returns the main data of the WAL record (xl_heap_delete, xl_heap_insert, xl_heap_multi_insert, xl_heap_update and so on). > * I don't think we need the stats at all. We can run GROUP BY queries > on the results of pg_get_wal_records_info(). As identified in [1], SQL-version of stats function is way slower in normal cases, hence it was agreed (by Andres, Kyotaro-san and myself) to have a C-function for stats. > * Include the xlinfo portion of the wal record in addition to the rmgr, > like pg_waldump --stats=record shows. That way we can GROUP BY that as > well. > > 5. For pg_get_wal_record and pg_get_wal_records, also return the xlinfo > using rm_identify() if available. Yes, that's already part of the description column (much like pg_waldump does) and the users can still do it with GROUP BY and HAVING clauses, see [4]. > * I don't think we need the changes to xlogutils.c. You calculate the > end pointer based on the flush pointer, anyway, so we should never need > to wait (and when I take it out, the tests still pass). > > 6. Remove changes to xlogutils. As mentioned in [1], read_local_xlog_page_no_wait required because the functions can still wait in read_local_xlog_page for WAL while finding the first valid record after the given input LSN (the use case is simple - just provide input LSN closer to server's current flush LSN, may be off by 3 or 4 bytes). Also, I tried to keep the changes minimal with the read_local_xlog_page_guts static function. IMO, that shouldn't be a problem. > I think we can radically simplify it without losing functionality, > unless I'm missing something. > > 1. Eliminate pg_get_wal_record(), > pg_get_wal_records_info_till_end_of_wal(), pg_get_wal_stats(), > pg_get_wal_stats_till_end_of_wal(). > > 4. For pg_get_wal_records, if end_lsn is NULL, read until the end of > WAL. It's pretty much clear to the users with till_end_of_wal functions instead of cooking many things into the same functions with default values for input LSNs as NULL which also requires the functions to be "CALLED ON NULL INPUT" types which isn't good. This was also suggested by Andres, see [2], and I agree with it. > 2. Rename pg_get_wal_record_info -> pg_get_wal_record > > 3. Rename pg_get_wal_records_info -> pg_get_wal_records As these functions aren't returning the WAL record data, but info about it (decoded data), I would like to retain the function names as-is. > 8. With only two functions in the API, it may even make sense to just > make it a part of postgres rather than a separate module. As said above, I would like to have till_end_of_wal versions. Firstly, pg_walinspect functions may not be needed by everyone, the extension provides a way for the users to install if required. Also, many hackers have suggested new functions [3], but, right now the idea is to get pg_walinspect onboard with simple-yet-useful functions and then think of extending it with new functions later. [1] https://www.postgresql.org/message-id/CALj2ACUvU2fGLw7keEpxZhGAoMQ9vrCPX-13hexQPoR%2BQRbuOw%40mail.g... [2] https://www.postgresql.org/message-id/20220322180006.hgbsoldgbljyrcm7%40alap3.anarazel.de [3] There are many functions we can add to pg_walinspect - functions with wait mode for future WAL, WAL parsing, function to return all the WAL record info/stats given a WAL file name, functions to return WAL info/stats from historic timelines as well, function to see if the given WAL file is valid and so on. [4] postgres=# select count(resource_manager), description, from pg_get_wal_records_info('0/14E0568', '0/14F2568') group by description having description like '%INSERT_LEAF%'; count | description -------+--------------------- 7 | INSERT_LEAF off 108 1 | INSERT_LEAF off 111 1 | INSERT_LEAF off 135 1 | INSERT_LEAF off 142 3 | INSERT_LEAF off 143 1 | INSERT_LEAF off 144 1 | INSERT_LEAF off 145 1 | INSERT_LEAF off 146 1 | INSERT_LEAF off 274 1 | INSERT_LEAF off 405 (10 rows) [5] /* * The overall layout of an XLOG record is: * Fixed-size header (XLogRecord struct) * XLogRecordBlockHeader struct * XLogRecordBlockHeader struct * ... * XLogRecordDataHeader[Short|Long] struct * block data * block data * ... * main data [6] XLogRecord * XLogReadRecord(XLogReaderState *state, char **errormsg) { decoded = XLogNextRecord(state, errormsg); if (decoded) { /* * This function returns a pointer to the record's header, not the * actual decoded record. The caller will access the decoded record * through the XLogRecGetXXX() macros, which reach the decoded * recorded as xlogreader->record. */ Assert(state->record == decoded); return &decoded->header; } [7] https://www.postgresql.org/message-id/20220322180006.hgbsoldgbljyrcm7%40alap3.anarazel.de Regards, Bharath Rupireddy. ^ permalink raw reply [nested|flat] 9+ messages in thread
* Re: pg_walinspect - a new extension to get raw WAL data and WAL stats 2022-04-04 03:45 Re: pg_walinspect - a new extension to get raw WAL data and WAL stats Bharath Rupireddy <[email protected]> @ 2022-04-06 05:02 ` Jeff Davis <[email protected]> 2022-04-06 08:45 ` Re: pg_walinspect - a new extension to get raw WAL data and WAL stats Bharath Rupireddy <[email protected]> 0 siblings, 1 reply; 9+ messages in thread From: Jeff Davis @ 2022-04-06 05:02 UTC (permalink / raw) To: Bharath Rupireddy <[email protected]>; +Cc: RKN Sai Krishna <[email protected]>; Andres Freund <[email protected]>; Kyotaro Horiguchi <[email protected]>; Ashutosh Sharma <[email protected]>; Stephen Frost <[email protected]>; Robert Haas <[email protected]>; Andrew Dunstan <[email protected]>; Greg Stark <[email protected]>; Jeremy Schneider <[email protected]>; Bruce Momjian <[email protected]>; PostgreSQL Hackers <[email protected]>; SATYANARAYANA NARLAPURAM <[email protected]>; [email protected]; [email protected] On Mon, 2022-04-04 at 09:15 +0530, Bharath Rupireddy wrote: > My intention is to return the overall undecoded WAL record [5] i.e. > the data starting from XLogReadRecord's output [6] till length > XLogRecGetTotalLen(xlogreader);. Please see [7], where Andres agreed > to have this function, I also mentioned a possible use-case there. The current patch does not actually do this: it's returning a pointer into the DecodedXLogRecord struct, which doesn't have the raw bytes of the WAL record. To return the raw bytes of the record is not entirely trivial: it seems we have to look in the decoded record and either find a pointer into readBuf, or readRecordBuf, depending on whether the record crosses a boundary or not. If we find a good way to do this I'm fine keeping the function, but if not, we can leave it for v16. > pg_get_wal_record_info returns the main data of the WAL record > (xl_heap_delete, xl_heap_insert, xl_heap_multi_insert, xl_heap_update > and so on). We also discussed just removing the main data from the output here. It's not terribly useful, and could be arbitrarily large. Similar to how we leave out the backup block data and images. > As identified in [1], SQL-version of stats function is way slower in > normal cases, hence it was agreed (by Andres, Kyotaro-san and myself) > to have a C-function for stats.a pointer into Now I agree. We should also have an equivalent of "pg_waldump -- stats=record" though, too. > Yes, that's already part of the description column (much like > pg_waldump does) and the users can still do it with GROUP BY and > HAVING clauses, see [4]. I still think an extra column for the results of rm_identify() would make sense. Not critical, but seems useful. > As mentioned in [1], read_local_xlog_page_no_wait required because > the > functions can still wait in read_local_xlog_page for WAL while > finding > the first valid record after the given input LSN (the use case is > simple - just provide input LSN closer to server's current flush LSN, > may be off by 3 or 4 bytes). Did you look into using XLogReadAhead() rather than XLogReadRecord()? > It's pretty much clear to the users with till_end_of_wal functions > instead of cooking many things into the same functions with default > values for input LSNs as NULL which also requires the functions to be > "CALLED ON NULL INPUT" types which isn't good. This was also > suggested > by Andres, see [2], and I agree with it. OK, it's a matter of taste I suppose. I don't have a strong opinion. > > 2. Rename pg_get_wal_record_info -> pg_get_wal_record > > > > 3. Rename pg_get_wal_records_info -> pg_get_wal_records > > As these functions aren't returning the WAL record data, but info > about it (decoded data), I would like to retain the function names > as-is. The name pg_get_wal_records_info bothers me slightly, but I don't have a better suggestion. One other thought: functions like pg_logical_emit_message() return an LSN, but if you feed that into pg_walinspect you get the *next* record. That makes sense because pg_logical_emit_message() returns the result of XLogInsertRecord(), which is the end of the last inserted record. But it can be slightly annoying/confusing. I don't have any particular suggestion, but maybe it's worth a mention in the docs or something? Regards, Jeff Davis ^ permalink raw reply [nested|flat] 9+ messages in thread
* Re: pg_walinspect - a new extension to get raw WAL data and WAL stats 2022-04-04 03:45 Re: pg_walinspect - a new extension to get raw WAL data and WAL stats Bharath Rupireddy <[email protected]> 2022-04-06 05:02 ` Re: pg_walinspect - a new extension to get raw WAL data and WAL stats Jeff Davis <[email protected]> @ 2022-04-06 08:45 ` Bharath Rupireddy <[email protected]> 2022-04-07 10:05 ` Re: pg_walinspect - a new extension to get raw WAL data and WAL stats Bharath Rupireddy <[email protected]> 0 siblings, 1 reply; 9+ messages in thread From: Bharath Rupireddy @ 2022-04-06 08:45 UTC (permalink / raw) To: Jeff Davis <[email protected]>; +Cc: RKN Sai Krishna <[email protected]>; Andres Freund <[email protected]>; Kyotaro Horiguchi <[email protected]>; Ashutosh Sharma <[email protected]>; Stephen Frost <[email protected]>; Robert Haas <[email protected]>; Andrew Dunstan <[email protected]>; Greg Stark <[email protected]>; Jeremy Schneider <[email protected]>; Bruce Momjian <[email protected]>; PostgreSQL Hackers <[email protected]>; SATYANARAYANA NARLAPURAM <[email protected]>; [email protected]; [email protected] On Wed, Apr 6, 2022 at 10:32 AM Jeff Davis <[email protected]> wrote: > > On Mon, 2022-04-04 at 09:15 +0530, Bharath Rupireddy wrote: > > My intention is to return the overall undecoded WAL record [5] i.e. > > the data starting from XLogReadRecord's output [6] till length > > XLogRecGetTotalLen(xlogreader);. Please see [7], where Andres agreed > > to have this function, I also mentioned a possible use-case there. > > The current patch does not actually do this: it's returning a pointer > into the DecodedXLogRecord struct, which doesn't have the raw bytes of > the WAL record. > > To return the raw bytes of the record is not entirely trivial: it seems > we have to look in the decoded record and either find a pointer into > readBuf, or readRecordBuf, depending on whether the record crosses a > boundary or not. If we find a good way to do this I'm fine keeping the > function, but if not, we can leave it for v16. With no immediate use of raw WAL data without a WAL record parsing function, I'm dropping that function for now. > > pg_get_wal_record_info returns the main data of the WAL record > > (xl_heap_delete, xl_heap_insert, xl_heap_multi_insert, xl_heap_update > > and so on). > > We also discussed just removing the main data from the output here. > It's not terribly useful, and could be arbitrarily large. Similar to > how we leave out the backup block data and images. Done. > > As identified in [1], SQL-version of stats function is way slower in > > normal cases, hence it was agreed (by Andres, Kyotaro-san and myself) > > to have a C-function for stats.a pointer into > > Now I agree. We should also have an equivalent of "pg_waldump -- > stats=record" though, too. Added a parameter per_record (with default being false, emitting per-rmgr stats) to pg_get_wal_stats and pg_get_wal_stats_till_end_of_wal, when set returns per-record stats, much like pg_waldump. > > Yes, that's already part of the description column (much like > > pg_waldump does) and the users can still do it with GROUP BY and > > HAVING clauses, see [4]. > > I still think an extra column for the results of rm_identify() would > make sense. Not critical, but seems useful. Added rm_identify as record_type column in pg_get_wal_record_info, pg_get_wal_records_info, pg_get_wal_record_info_till_end_of_wal. Removed the rm_identify from the description column as it's unnecessary now here. > > As mentioned in [1], read_local_xlog_page_no_wait required because > > the > > functions can still wait in read_local_xlog_page for WAL while > > finding > > the first valid record after the given input LSN (the use case is > > simple - just provide input LSN closer to server's current flush LSN, > > may be off by 3 or 4 bytes). > > Did you look into using XLogReadAhead() rather than XLogReadRecord()? I don't think XLogReadAhead will help either, as it calls page_read callback, XLogReadAhead->XLogDecodeNextRecord->ReadPageInternal->page_read->read_local_xlog_page (which again waits for future WAL). Per our internal discussion, I'm keeping the read_local_xlog_page_no_wait as it offers a better solution without much code duplication. > The name pg_get_wal_records_info bothers me slightly, but I don't have > a better suggestion. IMO, pg_get_wal_records_info looks okay, hence didn't change it. > One other thought: functions like pg_logical_emit_message() return an > LSN, but if you feed that into pg_walinspect you get the *next* record. > That makes sense because pg_logical_emit_message() returns the result > of XLogInsertRecord(), which is the end of the last inserted record. > But it can be slightly annoying/confusing. I don't have any particular > suggestion, but maybe it's worth a mention in the docs or something? Yes, all the pg_walinspect functions would find the next valid WAL record to the input/start LSN and start returning the details from then. IMO, the descriptions of these functions have already specified it: pg_get_wal_record_info Gets WAL record information of a given LSN. If the given LSN isn't containing a valid WAL record, it gives the information of the next available valid WAL record. This function emits an error if a future (the all other functions say this: Gets information/statistics of all the valid WAL records between/from Attaching v17 patch-set with the above review comments addressed. Please have a look at it. Regards, Bharath Rupireddy. Attachments: [application/octet-stream] v17-0001-Refactor-pg_waldump-code.patch (18.3K, ../../CALj2ACXC8jmF1d74Mb9Ca9sVsN4biVDTka3=x6XO22XDM-t1jw@mail.gmail.com/2-v17-0001-Refactor-pg_waldump-code.patch) download | inline diff: From bdb838c0538b30b798e58424fb06498a9acb008d Mon Sep 17 00:00:00 2001 From: Bharath Rupireddy <[email protected]> Date: Wed, 6 Apr 2022 02:56:20 +0000 Subject: [PATCH v17] Refactor pg_waldump code This patch puts some generic chunks of pg_waldump's code into separate reusable functions in xlogdesc.c and xlogstats.c, a new file along xlogstats.h introduced for placing WAL stats and structures. This way, other modules can reuse these common functions. --- src/backend/access/rmgrdesc/xlogdesc.c | 125 ++++++++++++++++ src/backend/access/transam/Makefile | 1 + src/backend/access/transam/xlogstats.c | 93 ++++++++++++ src/bin/pg_waldump/.gitignore | 1 + src/bin/pg_waldump/Makefile | 8 +- src/bin/pg_waldump/pg_waldump.c | 198 ++----------------------- src/include/access/xlog_internal.h | 5 + src/include/access/xlogstats.h | 40 +++++ 8 files changed, 282 insertions(+), 189 deletions(-) create mode 100644 src/backend/access/transam/xlogstats.c create mode 100644 src/include/access/xlogstats.h diff --git a/src/backend/access/rmgrdesc/xlogdesc.c b/src/backend/access/rmgrdesc/xlogdesc.c index e7452af679..429e5dcd5b 100644 --- a/src/backend/access/rmgrdesc/xlogdesc.c +++ b/src/backend/access/rmgrdesc/xlogdesc.c @@ -200,3 +200,128 @@ xlog_identify(uint8 info) return id; } + +/* + * Returns a string giving information about all the blocks in an + * XLogRecord. + */ +void +XLogRecGetBlockRefInfo(XLogReaderState *record, char *delimiter, + uint32 *fpi_len, bool detailed_format, + StringInfo buf) +{ + int block_id; + + Assert(record != NULL); + + if (detailed_format && delimiter != NULL) + appendStringInfoChar(buf, '\n'); + + for (block_id = 0; block_id <= XLogRecMaxBlockId(record); block_id++) + { + RelFileNode rnode = {InvalidOid, InvalidOid, InvalidOid}; + ForkNumber forknum = InvalidForkNumber; + BlockNumber blk = InvalidBlockNumber; + + if (!XLogRecHasBlockRef(record, block_id)) + continue; + + XLogRecGetBlockTag(record, block_id, &rnode, &forknum, &blk); + + if (detailed_format) + { + /* Get block references in detailed format. */ + + appendStringInfo(buf, + "\tblkref #%d: rel %u/%u/%u fork %s blk %u", + block_id, + rnode.spcNode, rnode.dbNode, rnode.relNode, + forkNames[forknum], + blk); + + if (XLogRecHasBlockImage(record, block_id)) + { + uint8 bimg_info = XLogRecGetBlock(record, block_id)->bimg_info; + + /* Calculate the amount of FPI data in the record. */ + if (fpi_len) + *fpi_len += XLogRecGetBlock(record, block_id)->bimg_len; + + if (BKPIMAGE_COMPRESSED(bimg_info)) + { + const char *method; + + if ((bimg_info & BKPIMAGE_COMPRESS_PGLZ) != 0) + method = "pglz"; + else if ((bimg_info & BKPIMAGE_COMPRESS_LZ4) != 0) + method = "lz4"; + else if ((bimg_info & BKPIMAGE_COMPRESS_ZSTD) != 0) + method = "zstd"; + else + method = "unknown"; + + appendStringInfo(buf, + " (FPW%s); hole: offset: %u, length: %u, " + "compression saved: %u, method: %s", + XLogRecBlockImageApply(record, block_id) ? + "" : " for WAL verification", + XLogRecGetBlock(record, block_id)->hole_offset, + XLogRecGetBlock(record, block_id)->hole_length, + BLCKSZ - + XLogRecGetBlock(record, block_id)->hole_length - + XLogRecGetBlock(record, block_id)->bimg_len, + method); + } + else + { + appendStringInfo(buf, + " (FPW%s); hole: offset: %u, length: %u", + XLogRecBlockImageApply(record, block_id) ? + "" : " for WAL verification", + XLogRecGetBlock(record, block_id)->hole_offset, + XLogRecGetBlock(record, block_id)->hole_length); + } + } + } + else + { + /* Get block references in short format. */ + + if (forknum != MAIN_FORKNUM) + { + appendStringInfo(buf, + ", blkref #%d: rel %u/%u/%u fork %s blk %u", + block_id, + rnode.spcNode, rnode.dbNode, rnode.relNode, + forkNames[forknum], + blk); + } + else + { + appendStringInfo(buf, + ", blkref #%d: rel %u/%u/%u blk %u", + block_id, + rnode.spcNode, rnode.dbNode, rnode.relNode, + blk); + } + + if (XLogRecHasBlockImage(record, block_id)) + { + /* Calculate the amount of FPI data in the record. */ + if (fpi_len) + *fpi_len += XLogRecGetBlock(record, block_id)->bimg_len; + + if (XLogRecBlockImageApply(record, block_id)) + appendStringInfo(buf, " FPW"); + else + appendStringInfo(buf, " FPW for WAL verification"); + } + } + + if (detailed_format && delimiter != NULL) + appendStringInfoChar(buf, '\n'); + } + + if (!detailed_format && delimiter != NULL) + appendStringInfoChar(buf, '\n'); +} diff --git a/src/backend/access/transam/Makefile b/src/backend/access/transam/Makefile index 79314c69ab..071f3dbe0f 100644 --- a/src/backend/access/transam/Makefile +++ b/src/backend/access/transam/Makefile @@ -33,6 +33,7 @@ OBJS = \ xloginsert.o \ xlogreader.o \ xlogrecovery.o \ + xlogstats.o \ xlogutils.o include $(top_srcdir)/src/backend/common.mk diff --git a/src/backend/access/transam/xlogstats.c b/src/backend/access/transam/xlogstats.c new file mode 100644 index 0000000000..aff3069ecb --- /dev/null +++ b/src/backend/access/transam/xlogstats.c @@ -0,0 +1,93 @@ +/*------------------------------------------------------------------------- + * + * xlogstats.c + * Functions for WAL Statitstics + * + * Copyright (c) 2022, PostgreSQL Global Development Group + * + * IDENTIFICATION + * src/backend/access/transam/xlogstats.c + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include "access/xlogreader.h" +#include "access/xlogstats.h" + +/* + * Calculate the size of a record, split into !FPI and FPI parts. + */ +void +XLogRecGetLen(XLogReaderState *record, uint32 *rec_len, + uint32 *fpi_len) +{ + int block_id; + + /* + * Calculate the amount of FPI data in the record. + * + * XXX: We peek into xlogreader's private decoded backup blocks for the + * bimg_len indicating the length of FPI data. + */ + *fpi_len = 0; + for (block_id = 0; block_id <= XLogRecMaxBlockId(record); block_id++) + { + if (XLogRecHasBlockImage(record, block_id)) + *fpi_len += XLogRecGetBlock(record, block_id)->bimg_len; + } + + /* + * Calculate the length of the record as the total length - the length of + * all the block images. + */ + *rec_len = XLogRecGetTotalLen(record) - *fpi_len; +} + +/* + * Store per-rmgr and per-record statistics for a given record. + */ +void +XLogRecStoreStats(XLogStats *stats, XLogReaderState *record) +{ + RmgrId rmid; + uint8 recid; + uint32 rec_len; + uint32 fpi_len; + + Assert(stats != NULL && record != NULL); + + stats->count++; + + rmid = XLogRecGetRmid(record); + + XLogRecGetLen(record, &rec_len, &fpi_len); + + /* Update per-rmgr statistics */ + + stats->rmgr_stats[rmid].count++; + stats->rmgr_stats[rmid].rec_len += rec_len; + stats->rmgr_stats[rmid].fpi_len += fpi_len; + + /* + * Update per-record statistics, where the record is identified by a + * combination of the RmgrId and the four bits of the xl_info field that + * are the rmgr's domain (resulting in sixteen possible entries per + * RmgrId). + */ + + recid = XLogRecGetInfo(record) >> 4; + + /* + * XACT records need to be handled differently. Those records use the + * first bit of those four bits for an optional flag variable and the + * following three bits for the opcode. We filter opcode out of xl_info + * and use it as the identifier of the record. + */ + if (rmid == RM_XACT_ID) + recid &= 0x07; + + stats->record_stats[rmid][recid].count++; + stats->record_stats[rmid][recid].rec_len += rec_len; + stats->record_stats[rmid][recid].fpi_len += fpi_len; +} diff --git a/src/bin/pg_waldump/.gitignore b/src/bin/pg_waldump/.gitignore index 3be00a8b61..dabb6e34b6 100644 --- a/src/bin/pg_waldump/.gitignore +++ b/src/bin/pg_waldump/.gitignore @@ -23,6 +23,7 @@ /xactdesc.c /xlogdesc.c /xlogreader.c +/xlogstat.c # Generated by test suite /tmp_check/ diff --git a/src/bin/pg_waldump/Makefile b/src/bin/pg_waldump/Makefile index 9f333d0c8a..d6459e17c7 100644 --- a/src/bin/pg_waldump/Makefile +++ b/src/bin/pg_waldump/Makefile @@ -13,7 +13,8 @@ OBJS = \ compat.o \ pg_waldump.o \ rmgrdesc.o \ - xlogreader.o + xlogreader.o \ + xlogstats.o override CPPFLAGS := -DFRONTEND $(CPPFLAGS) @@ -29,6 +30,9 @@ pg_waldump: $(OBJS) | submake-libpgport xlogreader.c: % : $(top_srcdir)/src/backend/access/transam/% rm -f $@ && $(LN_S) $< . +xlogstats.c: % : $(top_srcdir)/src/backend/access/transam/% + rm -f $@ && $(LN_S) $< . + $(RMGRDESCSOURCES): % : $(top_srcdir)/src/backend/access/rmgrdesc/% rm -f $@ && $(LN_S) $< . @@ -42,7 +46,7 @@ uninstall: rm -f '$(DESTDIR)$(bindir)/pg_waldump$(X)' clean distclean maintainer-clean: - rm -f pg_waldump$(X) $(OBJS) $(RMGRDESCSOURCES) xlogreader.c + rm -f pg_waldump$(X) $(OBJS) $(RMGRDESCSOURCES) xlogreader.c xlogstats.c rm -rf tmp_check check: diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c index 4cb40d068a..61f06e828f 100644 --- a/src/bin/pg_waldump/pg_waldump.c +++ b/src/bin/pg_waldump/pg_waldump.c @@ -21,6 +21,7 @@ #include "access/xlog_internal.h" #include "access/xlogreader.h" #include "access/xlogrecord.h" +#include "access/xlogstats.h" #include "common/fe_memutils.h" #include "common/logging.h" #include "getopt_long.h" @@ -66,23 +67,6 @@ typedef struct XLogDumpConfig bool filter_by_fpw; } XLogDumpConfig; -typedef struct Stats -{ - uint64 count; - uint64 rec_len; - uint64 fpi_len; -} Stats; - -#define MAX_XLINFO_TYPES 16 - -typedef struct XLogDumpStats -{ - uint64 count; - XLogRecPtr startptr; - XLogRecPtr endptr; - Stats rmgr_stats[RM_NEXT_ID]; - Stats record_stats[RM_NEXT_ID][MAX_XLINFO_TYPES]; -} XLogDumpStats; #define fatal_error(...) do { pg_log_fatal(__VA_ARGS__); exit(EXIT_FAILURE); } while(0) @@ -453,81 +437,6 @@ XLogRecordHasFPW(XLogReaderState *record) return false; } -/* - * Calculate the size of a record, split into !FPI and FPI parts. - */ -static void -XLogDumpRecordLen(XLogReaderState *record, uint32 *rec_len, uint32 *fpi_len) -{ - int block_id; - - /* - * Calculate the amount of FPI data in the record. - * - * XXX: We peek into xlogreader's private decoded backup blocks for the - * bimg_len indicating the length of FPI data. - */ - *fpi_len = 0; - for (block_id = 0; block_id <= XLogRecMaxBlockId(record); block_id++) - { - if (XLogRecHasBlockImage(record, block_id)) - *fpi_len += XLogRecGetBlock(record, block_id)->bimg_len; - } - - /* - * Calculate the length of the record as the total length - the length of - * all the block images. - */ - *rec_len = XLogRecGetTotalLen(record) - *fpi_len; -} - -/* - * Store per-rmgr and per-record statistics for a given record. - */ -static void -XLogDumpCountRecord(XLogDumpConfig *config, XLogDumpStats *stats, - XLogReaderState *record) -{ - RmgrId rmid; - uint8 recid; - uint32 rec_len; - uint32 fpi_len; - - stats->count++; - - rmid = XLogRecGetRmid(record); - - XLogDumpRecordLen(record, &rec_len, &fpi_len); - - /* Update per-rmgr statistics */ - - stats->rmgr_stats[rmid].count++; - stats->rmgr_stats[rmid].rec_len += rec_len; - stats->rmgr_stats[rmid].fpi_len += fpi_len; - - /* - * Update per-record statistics, where the record is identified by a - * combination of the RmgrId and the four bits of the xl_info field that - * are the rmgr's domain (resulting in sixteen possible entries per - * RmgrId). - */ - - recid = XLogRecGetInfo(record) >> 4; - - /* - * XACT records need to be handled differently. Those records use the - * first bit of those four bits for an optional flag variable and the - * following three bits for the opcode. We filter opcode out of xl_info - * and use it as the identifier of the record. - */ - if (rmid == RM_XACT_ID) - recid &= 0x07; - - stats->record_stats[rmid][recid].count++; - stats->record_stats[rmid][recid].rec_len += rec_len; - stats->record_stats[rmid][recid].fpi_len += fpi_len; -} - /* * Print a record to stdout */ @@ -538,15 +447,12 @@ XLogDumpDisplayRecord(XLogDumpConfig *config, XLogReaderState *record) const RmgrDescData *desc = &RmgrDescTable[XLogRecGetRmid(record)]; uint32 rec_len; uint32 fpi_len; - RelFileNode rnode; - ForkNumber forknum; - BlockNumber blk; - int block_id; uint8 info = XLogRecGetInfo(record); XLogRecPtr xl_prev = XLogRecGetPrev(record); StringInfoData s; + char delim = {'\n'}; - XLogDumpRecordLen(record, &rec_len, &fpi_len); + XLogRecGetLen(record, &rec_len, &fpi_len); printf("rmgr: %-11s len (rec/tot): %6u/%6u, tx: %10u, lsn: %X/%08X, prev %X/%08X, ", desc->rm_name, @@ -564,93 +470,11 @@ XLogDumpDisplayRecord(XLogDumpConfig *config, XLogReaderState *record) initStringInfo(&s); desc->rm_desc(&s, record); printf("%s", s.data); - pfree(s.data); - - if (!config->bkp_details) - { - /* print block references (short format) */ - for (block_id = 0; block_id <= XLogRecMaxBlockId(record); block_id++) - { - if (!XLogRecHasBlockRef(record, block_id)) - continue; - - XLogRecGetBlockTag(record, block_id, &rnode, &forknum, &blk); - if (forknum != MAIN_FORKNUM) - printf(", blkref #%d: rel %u/%u/%u fork %s blk %u", - block_id, - rnode.spcNode, rnode.dbNode, rnode.relNode, - forkNames[forknum], - blk); - else - printf(", blkref #%d: rel %u/%u/%u blk %u", - block_id, - rnode.spcNode, rnode.dbNode, rnode.relNode, - blk); - if (XLogRecHasBlockImage(record, block_id)) - { - if (XLogRecBlockImageApply(record, block_id)) - printf(" FPW"); - else - printf(" FPW for WAL verification"); - } - } - putchar('\n'); - } - else - { - /* print block references (detailed format) */ - putchar('\n'); - for (block_id = 0; block_id <= XLogRecMaxBlockId(record); block_id++) - { - if (!XLogRecHasBlockRef(record, block_id)) - continue; - - XLogRecGetBlockTag(record, block_id, &rnode, &forknum, &blk); - printf("\tblkref #%d: rel %u/%u/%u fork %s blk %u", - block_id, - rnode.spcNode, rnode.dbNode, rnode.relNode, - forkNames[forknum], - blk); - if (XLogRecHasBlockImage(record, block_id)) - { - uint8 bimg_info = XLogRecGetBlock(record, block_id)->bimg_info; - if (BKPIMAGE_COMPRESSED(bimg_info)) - { - const char *method; - - if ((bimg_info & BKPIMAGE_COMPRESS_PGLZ) != 0) - method = "pglz"; - else if ((bimg_info & BKPIMAGE_COMPRESS_LZ4) != 0) - method = "lz4"; - else if ((bimg_info & BKPIMAGE_COMPRESS_ZSTD) != 0) - method = "zstd"; - else - method = "unknown"; - - printf(" (FPW%s); hole: offset: %u, length: %u, " - "compression saved: %u, method: %s", - XLogRecBlockImageApply(record, block_id) ? - "" : " for WAL verification", - XLogRecGetBlock(record, block_id)->hole_offset, - XLogRecGetBlock(record, block_id)->hole_length, - BLCKSZ - - XLogRecGetBlock(record, block_id)->hole_length - - XLogRecGetBlock(record, block_id)->bimg_len, - method); - } - else - { - printf(" (FPW%s); hole: offset: %u, length: %u", - XLogRecBlockImageApply(record, block_id) ? - "" : " for WAL verification", - XLogRecGetBlock(record, block_id)->hole_offset, - XLogRecGetBlock(record, block_id)->hole_length); - } - } - putchar('\n'); - } - } + resetStringInfo(&s); + XLogRecGetBlockRefInfo(record, &delim, NULL, config->bkp_details, &s); + printf("%s", s.data); + pfree(s.data); } /* @@ -698,7 +522,7 @@ XLogDumpStatsRow(const char *name, * Display summary statistics about the records seen so far. */ static void -XLogDumpDisplayStats(XLogDumpConfig *config, XLogDumpStats *stats) +XLogDumpDisplayStats(XLogDumpConfig *config, XLogStats *stats) { int ri, rj; @@ -859,7 +683,7 @@ main(int argc, char **argv) XLogReaderState *xlogreader_state; XLogDumpPrivate private; XLogDumpConfig config; - XLogDumpStats stats; + XLogStats stats; XLogRecord *record; XLogRecPtr first_record; char *waldir = NULL; @@ -913,7 +737,7 @@ main(int argc, char **argv) memset(&private, 0, sizeof(XLogDumpPrivate)); memset(&config, 0, sizeof(XLogDumpConfig)); - memset(&stats, 0, sizeof(XLogDumpStats)); + memset(&stats, 0, sizeof(XLogStats)); private.timeline = 1; private.startptr = InvalidXLogRecPtr; @@ -1289,7 +1113,7 @@ main(int argc, char **argv) { if (config.stats == true) { - XLogDumpCountRecord(&config, &stats, xlogreader_state); + XLogRecStoreStats(&stats, xlogreader_state); stats.endptr = xlogreader_state->EndRecPtr; } else diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index 0e94833129..d7c35c37c4 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -329,6 +329,11 @@ extern XLogRecPtr RequestXLogSwitch(bool mark_unimportant); extern void GetOldestRestartPoint(XLogRecPtr *oldrecptr, TimeLineID *oldtli); +extern void XLogRecGetBlockRefInfo(XLogReaderState *record, + char *delimiter, uint32 *fpi_len, + bool detailed_format, + StringInfo blk_ref); + /* * Exported for the functions in timeline.c and xlogarchive.c. Only valid * in the startup process. diff --git a/src/include/access/xlogstats.h b/src/include/access/xlogstats.h new file mode 100644 index 0000000000..36d833f82b --- /dev/null +++ b/src/include/access/xlogstats.h @@ -0,0 +1,40 @@ +/*------------------------------------------------------------------------- + * + * xlogstats.h + * Definitions for WAL Statitstics + * + * Copyright (c) 2022, PostgreSQL Global Development Group + * + * IDENTIFICATION + * src/include/access/xlogstats.h + * + *------------------------------------------------------------------------- + */ +#ifndef XLOGSTATS_H +#define XLOGSTATS_H + +#define MAX_XLINFO_TYPES 16 + +typedef struct XLogRecStats +{ + uint64 count; + uint64 rec_len; + uint64 fpi_len; +} XLogRecStats; + +typedef struct XLogStats +{ + uint64 count; +#ifdef FRONTEND + XLogRecPtr startptr; + XLogRecPtr endptr; +#endif + XLogRecStats rmgr_stats[RM_NEXT_ID]; + XLogRecStats record_stats[RM_NEXT_ID][MAX_XLINFO_TYPES]; +} XLogStats; + +extern void XLogRecGetLen(XLogReaderState *record, uint32 *rec_len, + uint32 *fpi_len); +extern void XLogRecStoreStats(XLogStats *stats, XLogReaderState *record); + +#endif /* XLOGSTATS_H */ -- 2.25.1 [application/octet-stream] v17-0002-pg_walinspect.patch (29.8K, ../../CALj2ACXC8jmF1d74Mb9Ca9sVsN4biVDTka3=x6XO22XDM-t1jw@mail.gmail.com/3-v17-0002-pg_walinspect.patch) download | inline diff: From 064f7f5c304558510b0f1769a6114df6eb98e55d Mon Sep 17 00:00:00 2001 From: Bharath Rupireddy <[email protected]> Date: Wed, 6 Apr 2022 07:56:10 +0000 Subject: [PATCH v17] pg_walinspect --- contrib/Makefile | 1 + contrib/pg_walinspect/.gitignore | 4 + contrib/pg_walinspect/Makefile | 26 + contrib/pg_walinspect/pg_walinspect--1.0.sql | 118 ++++ contrib/pg_walinspect/pg_walinspect.c | 639 +++++++++++++++++++ contrib/pg_walinspect/pg_walinspect.control | 5 + src/backend/access/transam/xlogreader.c | 13 +- src/backend/access/transam/xlogutils.c | 33 + src/bin/pg_waldump/pg_waldump.c | 5 + src/include/access/xlog.h | 2 +- src/include/access/xlog_internal.h | 2 +- src/include/access/xlogreader.h | 2 - src/include/access/xlogutils.h | 4 + 13 files changed, 843 insertions(+), 11 deletions(-) create mode 100644 contrib/pg_walinspect/.gitignore create mode 100644 contrib/pg_walinspect/Makefile create mode 100644 contrib/pg_walinspect/pg_walinspect--1.0.sql create mode 100644 contrib/pg_walinspect/pg_walinspect.c create mode 100644 contrib/pg_walinspect/pg_walinspect.control diff --git a/contrib/Makefile b/contrib/Makefile index 332b486ecc..bbf220407b 100644 --- a/contrib/Makefile +++ b/contrib/Makefile @@ -41,6 +41,7 @@ SUBDIRS = \ pgrowlocks \ pgstattuple \ pg_visibility \ + pg_walinspect \ postgres_fdw \ seg \ spi \ diff --git a/contrib/pg_walinspect/.gitignore b/contrib/pg_walinspect/.gitignore new file mode 100644 index 0000000000..5dcb3ff972 --- /dev/null +++ b/contrib/pg_walinspect/.gitignore @@ -0,0 +1,4 @@ +# Generated subdirectories +/log/ +/results/ +/tmp_check/ diff --git a/contrib/pg_walinspect/Makefile b/contrib/pg_walinspect/Makefile new file mode 100644 index 0000000000..c92a97447f --- /dev/null +++ b/contrib/pg_walinspect/Makefile @@ -0,0 +1,26 @@ +# contrib/pg_walinspect/Makefile + +MODULE_big = pg_walinspect +OBJS = \ + $(WIN32RES) \ + pg_walinspect.o +PGFILEDESC = "pg_walinspect - functions to inspect contents of PostgreSQL Write-Ahead Log" + +PG_CPPFLAGS = -I$(libpq_srcdir) +SHLIB_LINK_INTERNAL = $(libpq) + +EXTENSION = pg_walinspect +DATA = pg_walinspect--1.0.sql + +REGRESS = pg_walinspect + +ifdef USE_PGXS +PG_CONFIG = pg_config +PGXS := $(shell $(PG_CONFIG) --pgxs) +include $(PGXS) +else +subdir = contrib/pg_walinspect +top_builddir = ../.. +include $(top_builddir)/src/Makefile.global +include $(top_srcdir)/contrib/contrib-global.mk +endif diff --git a/contrib/pg_walinspect/pg_walinspect--1.0.sql b/contrib/pg_walinspect/pg_walinspect--1.0.sql new file mode 100644 index 0000000000..aae6456c18 --- /dev/null +++ b/contrib/pg_walinspect/pg_walinspect--1.0.sql @@ -0,0 +1,118 @@ +/* contrib/pg_walinspect/pg_walinspect--1.0.sql */ + +-- complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "CREATE EXTENSION pg_walinspect" to load this file. \quit + +-- +-- pg_get_wal_record_info() +-- +CREATE FUNCTION pg_get_wal_record_info(IN in_lsn pg_lsn, + OUT start_lsn pg_lsn, + OUT end_lsn pg_lsn, + OUT prev_lsn pg_lsn, + OUT xid xid, + OUT resource_manager text, + OUT record_type text, + OUT record_length int4, + OUT main_data_length int4, + OUT fpi_length int4, + OUT description text, + OUT block_ref text +) +AS 'MODULE_PATHNAME', 'pg_get_wal_record_info' +LANGUAGE C STRICT PARALLEL SAFE; + +REVOKE EXECUTE ON FUNCTION pg_get_wal_record_info(pg_lsn) FROM PUBLIC; +GRANT EXECUTE ON FUNCTION pg_get_wal_record_info(pg_lsn) TO pg_read_server_files; + +-- +-- pg_get_wal_records_info() +-- +CREATE FUNCTION pg_get_wal_records_info(IN start_lsn pg_lsn, + IN end_lsn pg_lsn, + OUT start_lsn pg_lsn, + OUT end_lsn pg_lsn, + OUT prev_lsn pg_lsn, + OUT xid xid, + OUT resource_manager text, + OUT record_type text, + OUT record_length int4, + OUT main_data_length int4, + OUT fpi_length int4, + OUT description text, + OUT block_ref text +) +RETURNS SETOF record +AS 'MODULE_PATHNAME', 'pg_get_wal_records_info' +LANGUAGE C STRICT PARALLEL SAFE; + +REVOKE EXECUTE ON FUNCTION pg_get_wal_records_info(pg_lsn, pg_lsn) FROM PUBLIC; +GRANT EXECUTE ON FUNCTION pg_get_wal_records_info(pg_lsn, pg_lsn) TO pg_read_server_files; + +-- +-- pg_get_wal_records_info_till_end_of_wal() +-- +CREATE FUNCTION pg_get_wal_records_info_till_end_of_wal(IN start_lsn pg_lsn, + OUT start_lsn pg_lsn, + OUT end_lsn pg_lsn, + OUT prev_lsn pg_lsn, + OUT xid xid, + OUT resource_manager text, + OUT record_type text, + OUT record_length int4, + OUT main_data_length int4, + OUT fpi_length int4, + OUT description text, + OUT block_ref text +) +RETURNS SETOF record +AS 'MODULE_PATHNAME', 'pg_get_wal_records_info_till_end_of_wal' +LANGUAGE C STRICT PARALLEL SAFE; + +REVOKE EXECUTE ON FUNCTION pg_get_wal_records_info_till_end_of_wal(pg_lsn) FROM PUBLIC; +GRANT EXECUTE ON FUNCTION pg_get_wal_records_info_till_end_of_wal(pg_lsn) TO pg_read_server_files; + +-- +-- pg_get_wal_stats() +-- +CREATE FUNCTION pg_get_wal_stats(IN start_lsn pg_lsn, + IN end_lsn pg_lsn, + IN per_record boolean DEFAULT false, + OUT "resource_manager/record_type" text, + OUT count int8, + OUT count_percentage float4, + OUT record_size int8, + OUT record_size_percentage float4, + OUT fpi_size int8, + OUT fpi_size_percentage float4, + OUT combined_size int8, + OUT combined_size_percentage float4 +) +RETURNS SETOF record +AS 'MODULE_PATHNAME', 'pg_get_wal_stats' +LANGUAGE C STRICT PARALLEL SAFE; + +REVOKE EXECUTE ON FUNCTION pg_get_wal_stats(pg_lsn, pg_lsn, boolean) FROM PUBLIC; +GRANT EXECUTE ON FUNCTION pg_get_wal_stats(pg_lsn, pg_lsn, boolean) TO pg_read_server_files; + +-- +-- pg_get_wal_stats_till_end_of_wal() +-- +CREATE FUNCTION pg_get_wal_stats_till_end_of_wal(IN start_lsn pg_lsn, + IN per_record boolean DEFAULT false, + OUT "resource_manager/record_type" text, + OUT count int8, + OUT count_percentage float4, + OUT record_size int8, + OUT record_size_percentage float4, + OUT fpi_size int8, + OUT fpi_size_percentage float4, + OUT combined_size int8, + OUT combined_size_percentage float4 +) +RETURNS SETOF record +AS 'MODULE_PATHNAME', 'pg_get_wal_stats_till_end_of_wal' +LANGUAGE C STRICT PARALLEL SAFE; + +REVOKE EXECUTE ON FUNCTION pg_get_wal_stats_till_end_of_wal(pg_lsn, boolean) FROM PUBLIC; +GRANT EXECUTE ON FUNCTION pg_get_wal_stats_till_end_of_wal(pg_lsn, boolean) TO pg_read_server_files; diff --git a/contrib/pg_walinspect/pg_walinspect.c b/contrib/pg_walinspect/pg_walinspect.c new file mode 100644 index 0000000000..b93e16752c --- /dev/null +++ b/contrib/pg_walinspect/pg_walinspect.c @@ -0,0 +1,639 @@ +/*------------------------------------------------------------------------- + * + * pg_walinspect.c + * Functions to inspect contents of PostgreSQL Write-Ahead Log + * + * Copyright (c) 2022, PostgreSQL Global Development Group + * + * IDENTIFICATION + * contrib/pg_walinspect/pg_walinspect.c + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include "access/xlog.h" +#include "access/xlog_internal.h" +#include "access/xlogreader.h" +#include "access/xlogrecovery.h" +#include "access/xlogstats.h" +#include "access/xlogutils.h" +#include "funcapi.h" +#include "miscadmin.h" +#include "utils/builtins.h" +#include "utils/pg_lsn.h" + +/* + * NOTE: For any code change or issue fix here, it is highly recommended to + * give a thought about doing the same in pg_waldump tool as well. + */ + +PG_MODULE_MAGIC; + +PG_FUNCTION_INFO_V1(pg_get_wal_record_info); +PG_FUNCTION_INFO_V1(pg_get_wal_records_info); +PG_FUNCTION_INFO_V1(pg_get_wal_records_info_till_end_of_wal); +PG_FUNCTION_INFO_V1(pg_get_wal_stats); +PG_FUNCTION_INFO_V1(pg_get_wal_stats_till_end_of_wal); + +/* + * Struct holding information about the parameters that can be passed to + * GetWalStats. + */ +typedef struct GetWalStatsParams +{ + /* If true, generate statistics per-record instead of per-rmgr. */ + bool stats_per_record; +} GetWalStatsParams; + +typedef void (*GetWALDetailsCB) (FunctionCallInfo fcinfo, + XLogRecPtr start_lsn, + XLogRecPtr end_lsn, + void *params); + +static bool IsFutureLSN(XLogRecPtr lsn, XLogRecPtr *curr_lsn); +static XLogReaderState *InitXLogReaderState(XLogRecPtr lsn, + XLogRecPtr *first_record); +static XLogRecord *ReadNextXLogRecord(XLogReaderState *xlogreader, + XLogRecPtr first_record); +static void GetWALRecordInfo(XLogReaderState *record, XLogRecPtr lsn, + Datum *values, bool *nulls, uint32 ncols); +static void GetWALDetailsGuts(FunctionCallInfo fcinfo, bool till_end_of_wal, + GetWALDetailsCB wal_details_cb); +static void GetWALRecordsInfo(FunctionCallInfo fcinfo, XLogRecPtr start_lsn, + XLogRecPtr end_lsn, void *params); +static void GetXLogSummaryStats(XLogStats * stats, ReturnSetInfo *rsinfo, + Datum *values, bool *nulls, uint32 ncols, + bool stats_per_record); +static void FillXLogStatsRow(const char *name, uint64 n, uint64 total_count, + uint64 rec_len, uint64 total_rec_len, + uint64 fpi_len, uint64 total_fpi_len, + uint64 tot_len, uint64 total_len, + Datum *values, bool *nulls, uint32 ncols); +static void GetWalStats(FunctionCallInfo fcinfo, XLogRecPtr start_lsn, + XLogRecPtr end_lsn, void *params); + +/* + * Check if the given LSN is in future. Also, return the LSN up to which the + * server has WAL. + */ +static bool +IsFutureLSN(XLogRecPtr lsn, XLogRecPtr *curr_lsn) +{ + /* + * We determine the current LSN of the server similar to how page_read + * callback read_local_xlog_page_no_wait does. + */ + if (!RecoveryInProgress()) + *curr_lsn = GetFlushRecPtr(NULL); + else + *curr_lsn = GetXLogReplayRecPtr(NULL); + + Assert(!XLogRecPtrIsInvalid(*curr_lsn)); + + if (lsn >= *curr_lsn) + return true; + + return false; +} + +/* + * Intialize WAL reader and identify first valid LSN. + */ +static XLogReaderState * +InitXLogReaderState(XLogRecPtr lsn, XLogRecPtr *first_record) +{ + XLogReaderState *xlogreader; + + /* + * Reading WAL below the first page of the first sgements isn't allowed. + * This is a bootstrap WAL page and the page_read callback fails to read + * it. + */ + if (lsn < XLOG_BLCKSZ) + ereport(ERROR, + (errmsg("could not read WAL at LSN %X/%X", + LSN_FORMAT_ARGS(lsn)))); + + xlogreader = XLogReaderAllocate(wal_segment_size, NULL, + XL_ROUTINE(.page_read = &read_local_xlog_page_no_wait, + .segment_open = &wal_segment_open, + .segment_close = &wal_segment_close), + NULL); + + if (xlogreader == NULL) + ereport(ERROR, + (errcode(ERRCODE_OUT_OF_MEMORY), + errmsg("out of memory"), + errdetail("Failed while allocating a WAL reading processor."))); + + /* First find a valid recptr to start from. */ + *first_record = XLogFindNextRecord(xlogreader, lsn); + + if (XLogRecPtrIsInvalid(*first_record)) + ereport(ERROR, + (errmsg("could not find a valid record after %X/%X", + LSN_FORMAT_ARGS(lsn)))); + + return xlogreader; +} + +/* + * Read next WAL record. + */ +static XLogRecord * +ReadNextXLogRecord(XLogReaderState *xlogreader, XLogRecPtr first_record) +{ + XLogRecord *record; + char *errormsg; + + record = XLogReadRecord(xlogreader, &errormsg); + + if (record == NULL) + { + if (errormsg) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not read WAL at %X/%X: %s", + LSN_FORMAT_ARGS(first_record), errormsg))); + else + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not read WAL at %X/%X", + LSN_FORMAT_ARGS(first_record)))); + } + + return record; +} + +/* + * Get a single WAL record info. + */ +static void +GetWALRecordInfo(XLogReaderState *record, XLogRecPtr lsn, + Datum *values, bool *nulls, uint32 ncols) +{ + const char *id; + const RmgrData *desc; + uint32 fpi_len = 0; + StringInfoData rec_desc; + StringInfoData rec_blk_ref; + uint32 main_data_len; + int i = 0; + + desc = &RmgrTable[XLogRecGetRmid(record)]; + id = desc->rm_identify(XLogRecGetInfo(record)); + + if (id == NULL) + id = psprintf("UNKNOWN (%x)", XLogRecGetInfo(record) & ~XLR_INFO_MASK); + + initStringInfo(&rec_desc); + desc->rm_desc(&rec_desc, record); + + /* Block references. */ + initStringInfo(&rec_blk_ref); + XLogRecGetBlockRefInfo(record, NULL, &fpi_len, true, &rec_blk_ref); + + main_data_len = XLogRecGetDataLen(record); + + values[i++] = LSNGetDatum(lsn); + values[i++] = LSNGetDatum(record->EndRecPtr - 1); + values[i++] = LSNGetDatum(XLogRecGetPrev(record)); + values[i++] = TransactionIdGetDatum(XLogRecGetXid(record)); + values[i++] = CStringGetTextDatum(desc->rm_name); + values[i++] = CStringGetTextDatum(id); + values[i++] = UInt32GetDatum(XLogRecGetTotalLen(record)); + values[i++] = UInt32GetDatum(main_data_len); + values[i++] = UInt32GetDatum(fpi_len); + values[i++] = CStringGetTextDatum(rec_desc.data); + values[i++] = CStringGetTextDatum(rec_blk_ref.data); + + Assert(i == ncols); +} + +/* + * Get WAL record info. + * + * This function emits an error if a future WAL LSN i.e. WAL LSN the database + * system doesn't know about is specified. + */ +Datum +pg_get_wal_record_info(PG_FUNCTION_ARGS) +{ +#define PG_GET_WAL_RECORD_INFO_COLS 11 + Datum result; + Datum values[PG_GET_WAL_RECORD_INFO_COLS]; + bool nulls[PG_GET_WAL_RECORD_INFO_COLS]; + XLogRecPtr lsn; + XLogRecPtr curr_lsn; + XLogRecPtr first_record; + XLogReaderState *xlogreader; + TupleDesc tupdesc; + HeapTuple tuple; + + lsn = PG_GETARG_LSN(0); + + if (IsFutureLSN(lsn, &curr_lsn)) + { + /* + * GetFlushRecPtr or GetXLogReplayRecPtr gives "end+1" LSN of the last + * record flushed or replayed respectively. But let's use the LSN up + * to "end" in user facing message. + */ + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("cannot accept future input LSN"), + errdetail("Last WAL record on the database system ends at LSN %X/%X.", + LSN_FORMAT_ARGS(curr_lsn - 1)))); + } + + /* Build a tuple descriptor for our result type. */ + if (get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_COMPOSITE) + elog(ERROR, "return type must be a row type"); + + xlogreader = InitXLogReaderState(lsn, &first_record); + + Assert(xlogreader); + + (void) ReadNextXLogRecord(xlogreader, first_record); + + MemSet(values, 0, sizeof(values)); + MemSet(nulls, 0, sizeof(nulls)); + + GetWALRecordInfo(xlogreader, first_record, values, nulls, + PG_GET_WAL_RECORD_INFO_COLS); + + XLogReaderFree(xlogreader); + + tuple = heap_form_tuple(tupdesc, values, nulls); + result = HeapTupleGetDatum(tuple); + + PG_RETURN_DATUM(result); +#undef PG_GET_WAL_RECORD_INFO_COLS +} + +/* + * Get WAL details such as record info, stats using the passed in callback. + */ +static void +GetWALDetailsGuts(FunctionCallInfo fcinfo, bool till_end_of_wal, + GetWALDetailsCB wal_details_cb) +{ + XLogRecPtr start_lsn; + XLogRecPtr end_lsn; + XLogRecPtr curr_lsn; + GetWalStatsParams stats_params; + + start_lsn = PG_GETARG_LSN(0); + + /* If not till end of wal, end_lsn would have been specified. */ + if (!till_end_of_wal) + end_lsn = PG_GETARG_LSN(1); + + if (IsFutureLSN(start_lsn, &curr_lsn)) + { + /* + * GetFlushRecPtr or GetXLogReplayRecPtr gives "end+1" LSN of the last + * record flushed or replayed respectively. But let's use the LSN up + * to "end" in user facing message. + */ + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("cannot accept future start LSN"), + errdetail("Last WAL record on the database system ends at LSN %X/%X.", + LSN_FORMAT_ARGS(curr_lsn - 1)))); + } + + if (!till_end_of_wal && end_lsn >= curr_lsn) + { + /* + * GetFlushRecPtr or GetXLogReplayRecPtr gives "end+1" LSN of the last + * record flushed or replayed respectively. But let's use the LSN up + * to "end" in user facing message. + */ + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("cannot accept future end LSN"), + errdetail("Last WAL record on the database system ends at LSN %X/%X.", + LSN_FORMAT_ARGS(curr_lsn - 1)))); + } + else if (till_end_of_wal) + { + /* + * GetFlushRecPtr or GetXLogReplayRecPtr gives "end+1" LSN of the last + * record flushed or replayed respectively. But let's use the LSN up to + * "end". + */ + end_lsn = curr_lsn - 1; + } + + if (start_lsn >= end_lsn) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("WAL start LSN must be less than end LSN"))); + + if (wal_details_cb == GetWalStats) + { + MemSet(&stats_params, 0, sizeof(GetWalStatsParams)); + + if (!till_end_of_wal) + stats_params.stats_per_record = PG_GETARG_BOOL(2); + else + stats_params.stats_per_record = PG_GETARG_BOOL(1); + + wal_details_cb(fcinfo, start_lsn, end_lsn, (void *) &stats_params); + } + else if (wal_details_cb == GetWALRecordsInfo) + { + wal_details_cb(fcinfo, start_lsn, end_lsn, NULL); + } +} + +/* + * Get info and data of all WAL records between start LSN and end LSN. + */ +static void +GetWALRecordsInfo(FunctionCallInfo fcinfo, XLogRecPtr start_lsn, + XLogRecPtr end_lsn, void *params) +{ +#define PG_GET_WAL_RECORDS_INFO_COLS 11 + XLogRecPtr first_record; + XLogReaderState *xlogreader; + ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; + Datum values[PG_GET_WAL_RECORDS_INFO_COLS]; + bool nulls[PG_GET_WAL_RECORDS_INFO_COLS]; + + SetSingleFuncCall(fcinfo, 0); + + xlogreader = InitXLogReaderState(start_lsn, &first_record); + + Assert(xlogreader); + + MemSet(values, 0, sizeof(values)); + MemSet(nulls, 0, sizeof(nulls)); + + for (;;) + { + (void) ReadNextXLogRecord(xlogreader, first_record); + + /* + * Let's not show the record info if it is spanning more than the + * end_lsn. EndRecPtr is "end+1" of the last read record, hence + * use "end" here. + */ + if ((xlogreader->EndRecPtr - 1) <= end_lsn) + { + GetWALRecordInfo(xlogreader, xlogreader->currRecPtr, values, nulls, + PG_GET_WAL_RECORDS_INFO_COLS); + + tuplestore_putvalues(rsinfo->setResult, rsinfo->setDesc, + values, nulls); + } + + /* Exit loop if read up to end_lsn. */ + if (xlogreader->EndRecPtr >= end_lsn) + break; + + CHECK_FOR_INTERRUPTS(); + } + + XLogReaderFree(xlogreader); + +#undef PG_GET_WAL_RECORDS_INFO_COLS +} + +/* + * Get info and data of all WAL records between start LSN and end LSN. + * + * This function emits an error if a future start or end WAL LSN i.e. WAL LSN + * the database system doesn't know about is specified. + */ +Datum +pg_get_wal_records_info(PG_FUNCTION_ARGS) +{ + GetWALDetailsGuts(fcinfo, false, GetWALRecordsInfo); + + PG_RETURN_VOID(); +} + +/* + * Get info and data of all WAL records from start LSN till end of WAL. + * + * This function emits an error if a future start i.e. WAL LSN the database + * system doesn't know about is specified. + */ +Datum +pg_get_wal_records_info_till_end_of_wal(PG_FUNCTION_ARGS) +{ + GetWALDetailsGuts(fcinfo, true, GetWALRecordsInfo); + + PG_RETURN_VOID(); +} + +/* + * Fill single row of record counts and sizes for an rmgr or record. + */ +static void +FillXLogStatsRow(const char *name, + uint64 n, uint64 total_count, + uint64 rec_len, uint64 total_rec_len, + uint64 fpi_len, uint64 total_fpi_len, + uint64 tot_len, uint64 total_len, + Datum *values, bool *nulls, uint32 ncols) +{ + double n_pct, + rec_len_pct, + fpi_len_pct, + tot_len_pct; + int i = 0; + + n_pct = 0; + if (total_count != 0) + n_pct = 100 * (double) n / total_count; + + rec_len_pct = 0; + if (total_rec_len != 0) + rec_len_pct = 100 * (double) rec_len / total_rec_len; + + fpi_len_pct = 0; + if (total_fpi_len != 0) + fpi_len_pct = 100 * (double) fpi_len / total_fpi_len; + + tot_len_pct = 0; + if (total_len != 0) + tot_len_pct = 100 * (double) tot_len / total_len; + + values[i++] = CStringGetTextDatum(name); + values[i++] = Int64GetDatum(n); + values[i++] = Float4GetDatum(n_pct); + values[i++] = Int64GetDatum(rec_len); + values[i++] = Float4GetDatum(rec_len_pct); + values[i++] = Int64GetDatum(fpi_len); + values[i++] = Float4GetDatum(fpi_len_pct); + values[i++] = Int64GetDatum(tot_len); + values[i++] = Float4GetDatum(tot_len_pct); + + Assert(i == ncols); +} + +/* + * Get summary statistics about the records seen so far. + */ +static void +GetXLogSummaryStats(XLogStats *stats, ReturnSetInfo *rsinfo, + Datum *values, bool *nulls, uint32 ncols, + bool stats_per_record) +{ + uint64 total_count = 0; + uint64 total_rec_len = 0; + uint64 total_fpi_len = 0; + uint64 total_len = 0; + int ri; + + /* + * Each row shows its percentages of the total, so make a first pass to + * calculate column totals. + */ + for (ri = 0; ri < RM_NEXT_ID; ri++) + { + total_count += stats->rmgr_stats[ri].count; + total_rec_len += stats->rmgr_stats[ri].rec_len; + total_fpi_len += stats->rmgr_stats[ri].fpi_len; + } + total_len = total_rec_len + total_fpi_len; + + for (ri = 0; ri < RM_NEXT_ID; ri++) + { + uint64 count; + uint64 rec_len; + uint64 fpi_len; + uint64 tot_len; + const RmgrData *desc = &RmgrTable[ri]; + + if (stats_per_record) + { + int rj; + + for (rj = 0; rj < MAX_XLINFO_TYPES; rj++) + { + const char *id; + + count = stats->record_stats[ri][rj].count; + rec_len = stats->record_stats[ri][rj].rec_len; + fpi_len = stats->record_stats[ri][rj].fpi_len; + tot_len = rec_len + fpi_len; + + /* Skip undefined combinations and ones that didn't occur */ + if (count == 0) + continue; + + /* the upper four bits in xl_info are the rmgr's */ + id = desc->rm_identify(rj << 4); + if (id == NULL) + id = psprintf("UNKNOWN (%x)", rj << 4); + + FillXLogStatsRow(psprintf("%s/%s", desc->rm_name, id), count, + total_count, rec_len, total_rec_len, fpi_len, + total_fpi_len, tot_len, total_len, + values, nulls, ncols); + + tuplestore_putvalues(rsinfo->setResult, rsinfo->setDesc, + values, nulls); + } + } + else + { + count = stats->rmgr_stats[ri].count; + rec_len = stats->rmgr_stats[ri].rec_len; + fpi_len = stats->rmgr_stats[ri].fpi_len; + tot_len = rec_len + fpi_len; + + FillXLogStatsRow(desc->rm_name, count, total_count, rec_len, + total_rec_len, fpi_len, total_fpi_len, tot_len, + total_len, values, nulls, ncols); + + tuplestore_putvalues(rsinfo->setResult, rsinfo->setDesc, + values, nulls); + } + } +} + +/* + * Get WAL stats between start LSN and end LSN. + */ +static void +GetWalStats(FunctionCallInfo fcinfo, XLogRecPtr start_lsn, + XLogRecPtr end_lsn, void *params) +{ +#define PG_GET_WAL_STATS_COLS 9 + XLogRecPtr first_record; + XLogReaderState *xlogreader; + XLogStats stats; + ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; + Datum values[PG_GET_WAL_STATS_COLS]; + bool nulls[PG_GET_WAL_STATS_COLS]; + + SetSingleFuncCall(fcinfo, 0); + + xlogreader = InitXLogReaderState(start_lsn, &first_record); + + MemSet(&stats, 0, sizeof(stats)); + + for (;;) + { + (void) ReadNextXLogRecord(xlogreader, first_record); + + /* + * Let's not show the record info if it is spanning more than the + * end_lsn. EndRecPtr is "end+1" of the last read record, hence + * use "end" here. + */ + if ((xlogreader->EndRecPtr - 1) <= end_lsn) + XLogRecStoreStats(&stats, xlogreader); + + /* Exit loop if read up to end_lsn. */ + if (xlogreader->EndRecPtr >= end_lsn) + break; + + CHECK_FOR_INTERRUPTS(); + } + + XLogReaderFree(xlogreader); + + MemSet(values, 0, sizeof(values)); + MemSet(nulls, 0, sizeof(nulls)); + + GetXLogSummaryStats(&stats, rsinfo, values, nulls, + PG_GET_WAL_STATS_COLS, + ((GetWalStatsParams *)params)->stats_per_record); + +#undef PG_GET_WAL_STATS_COLS +} + +/* + * Get stats of all WAL records between start LSN and end LSN. + * + * This function emits an error if a future start or end WAL LSN i.e. WAL LSN + * the database system doesn't know about is specified. + */ +Datum +pg_get_wal_stats(PG_FUNCTION_ARGS) +{ + GetWALDetailsGuts(fcinfo, false, GetWalStats); + + PG_RETURN_VOID(); +} + +/* + * Get stats of all WAL records from start LSN till end of WAL. + * + * This function emits an error if a future start i.e. WAL LSN the database + * system doesn't know about is specified. + */ +Datum +pg_get_wal_stats_till_end_of_wal(PG_FUNCTION_ARGS) +{ + GetWALDetailsGuts(fcinfo, true, GetWalStats); + + PG_RETURN_VOID(); +} diff --git a/contrib/pg_walinspect/pg_walinspect.control b/contrib/pg_walinspect/pg_walinspect.control new file mode 100644 index 0000000000..017e56a2bb --- /dev/null +++ b/contrib/pg_walinspect/pg_walinspect.control @@ -0,0 +1,5 @@ +# pg_walinspect extension +comment = 'functions to inspect contents of PostgreSQL Write-Ahead Log' +default_version = '1.0' +module_pathname = '$libdir/pg_walinspect' +relocatable = true diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index e437c42992..585c94c488 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -1320,13 +1320,6 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr, return true; } -#ifdef FRONTEND -/* - * Functions that are currently not needed in the backend, but are better - * implemented inside xlogreader.c because of the internal facilities available - * here. - */ - /* * Find the first record with an lsn >= RecPtr. * @@ -1447,6 +1440,12 @@ err: return InvalidXLogRecPtr; } +#ifdef FRONTEND +/* + * Functions that are currently not needed in the backend, but are better + * implemented inside xlogreader.c because of the internal facilities available + * here. + */ #endif /* FRONTEND */ /* diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c index a4dedc58b7..50159fd4cc 100644 --- a/src/backend/access/transam/xlogutils.c +++ b/src/backend/access/transam/xlogutils.c @@ -79,6 +79,10 @@ typedef struct xl_invalid_page static HTAB *invalid_page_tab = NULL; +static int +read_local_xlog_page_guts(XLogReaderState *state, XLogRecPtr targetPagePtr, + int reqLen, XLogRecPtr targetRecPtr, + char *cur_page, bool wait_for_wal); /* Report a reference to an invalid page */ static void @@ -851,6 +855,31 @@ wal_segment_close(XLogReaderState *state) int read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen, XLogRecPtr targetRecPtr, char *cur_page) +{ + return read_local_xlog_page_guts(state, targetPagePtr, reqLen, + targetRecPtr, cur_page, true); +} + +/* + * Same as read_local_xlog_page except that it doesn't wait for future WAL + * to be available. + */ +int +read_local_xlog_page_no_wait(XLogReaderState *state, XLogRecPtr targetPagePtr, + int reqLen, XLogRecPtr targetRecPtr, + char *cur_page) +{ + return read_local_xlog_page_guts(state, targetPagePtr, reqLen, + targetRecPtr, cur_page, false); +} + +/* + * Implementation of read_local_xlog_page and its no wait version. + */ +static int +read_local_xlog_page_guts(XLogReaderState *state, XLogRecPtr targetPagePtr, + int reqLen, XLogRecPtr targetRecPtr, + char *cur_page, bool wait_for_wal) { XLogRecPtr read_upto, loc; @@ -906,6 +935,10 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, if (loc <= read_upto) break; + /* If asked, let's not wait for future WAL. */ + if (!wait_for_wal) + break; + CHECK_FOR_INTERRUPTS(); pg_usleep(1000L); } diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c index 61f06e828f..9dd9f05204 100644 --- a/src/bin/pg_waldump/pg_waldump.c +++ b/src/bin/pg_waldump/pg_waldump.c @@ -27,6 +27,11 @@ #include "getopt_long.h" #include "rmgrdesc.h" +/* + * NOTE: For any code change or issue fix here, it is highly recommended to + * give a thought about doing the same in pg_walinspect contrib module as well. + */ + static const char *progname; static int WalSegSz; diff --git a/src/include/access/xlog.h b/src/include/access/xlog.h index 09f6464331..3e644372f9 100644 --- a/src/include/access/xlog.h +++ b/src/include/access/xlog.h @@ -31,7 +31,7 @@ extern XLogRecPtr XactLastRecEnd; extern PGDLLIMPORT XLogRecPtr XactLastCommitEnd; /* these variables are GUC parameters related to XLOG */ -extern int wal_segment_size; +extern PGDLLIMPORT int wal_segment_size; extern int min_wal_size_mb; extern int max_wal_size_mb; extern int wal_keep_size_mb; diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index d7c35c37c4..2985c75361 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -319,7 +319,7 @@ typedef struct RmgrData struct XLogRecordBuffer *buf); } RmgrData; -extern const RmgrData RmgrTable[]; +extern PGDLLIMPORT const RmgrData RmgrTable[]; /* * Exported to support xlog switching from checkpointer diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h index f4388cc9be..b4c7d93787 100644 --- a/src/include/access/xlogreader.h +++ b/src/include/access/xlogreader.h @@ -340,9 +340,7 @@ extern void XLogReaderSetDecodeBuffer(XLogReaderState *state, /* Position the XLogReader to given record */ extern void XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr); -#ifdef FRONTEND extern XLogRecPtr XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr); -#endif /* FRONTEND */ /* Return values from XLogPageReadCB. */ typedef enum XLogPageReadResult diff --git a/src/include/access/xlogutils.h b/src/include/access/xlogutils.h index 64708949db..22c2299d68 100644 --- a/src/include/access/xlogutils.h +++ b/src/include/access/xlogutils.h @@ -92,6 +92,10 @@ extern void FreeFakeRelcacheEntry(Relation fakerel); extern int read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen, XLogRecPtr targetRecPtr, char *cur_page); +extern int read_local_xlog_page_no_wait(XLogReaderState *state, + XLogRecPtr targetPagePtr, int reqLen, + XLogRecPtr targetRecPtr, + char *cur_page); extern void wal_segment_open(XLogReaderState *state, XLogSegNo nextSegNo, TimeLineID *tli_p); -- 2.25.1 [application/octet-stream] v17-0003-pg_walinspect-tests.patch (9.7K, ../../CALj2ACXC8jmF1d74Mb9Ca9sVsN4biVDTka3=x6XO22XDM-t1jw@mail.gmail.com/4-v17-0003-pg_walinspect-tests.patch) download | inline diff: From 66cf3eca33d413cdfa59a4c5a5d4e8b164c7d3a1 Mon Sep 17 00:00:00 2001 From: Bharath Rupireddy <[email protected]> Date: Wed, 6 Apr 2022 06:25:48 +0000 Subject: [PATCH v17] pg_walinspect tests --- .../pg_walinspect/expected/pg_walinspect.out | 145 ++++++++++++++++++ contrib/pg_walinspect/sql/pg_walinspect.sql | 107 +++++++++++++ 2 files changed, 252 insertions(+) create mode 100644 contrib/pg_walinspect/expected/pg_walinspect.out create mode 100644 contrib/pg_walinspect/sql/pg_walinspect.sql diff --git a/contrib/pg_walinspect/expected/pg_walinspect.out b/contrib/pg_walinspect/expected/pg_walinspect.out new file mode 100644 index 0000000000..53d3146fe1 --- /dev/null +++ b/contrib/pg_walinspect/expected/pg_walinspect.out @@ -0,0 +1,145 @@ +CREATE EXTENSION pg_walinspect; +CREATE TABLE sample_tbl(col1 int, col2 int); +SELECT pg_current_wal_lsn() AS wal_lsn1 \gset +INSERT INTO sample_tbl SELECT * FROM generate_series(1, 2); +SELECT pg_current_wal_lsn() AS wal_lsn2 \gset +INSERT INTO sample_tbl SELECT * FROM generate_series(1, 2); +-- =================================================================== +-- Tests for input validation +-- =================================================================== +SELECT COUNT(*) >= 0 AS ok FROM pg_get_wal_records_info(:'wal_lsn2', :'wal_lsn1'); -- ERROR +ERROR: WAL start LSN must be less than end LSN +SELECT COUNT(*) >= 0 AS ok FROM pg_get_wal_stats(:'wal_lsn2', :'wal_lsn1'); -- ERROR +ERROR: WAL start LSN must be less than end LSN +-- =================================================================== +-- Tests for all function executions +-- =================================================================== +SELECT COUNT(*) >= 0 AS ok FROM pg_get_wal_record_info(:'wal_lsn1'); + ok +---- + t +(1 row) + +SELECT COUNT(*) >= 0 AS ok FROM pg_get_wal_records_info(:'wal_lsn1', :'wal_lsn2'); + ok +---- + t +(1 row) + +SELECT COUNT(*) >= 0 AS ok FROM pg_get_wal_records_info_till_end_of_wal(:'wal_lsn1'); + ok +---- + t +(1 row) + +SELECT COUNT(*) >= 0 AS ok FROM pg_get_wal_stats(:'wal_lsn1', :'wal_lsn2'); + ok +---- + t +(1 row) + +SELECT COUNT(*) >= 0 AS ok FROM pg_get_wal_stats_till_end_of_wal(:'wal_lsn1'); + ok +---- + t +(1 row) + +-- =================================================================== +-- Tests for filtering out WAL records of a particular table +-- =================================================================== +SELECT oid AS sample_tbl_oid FROM pg_class WHERE relname = 'sample_tbl' \gset +SELECT COUNT(*) >= 1 AS ok FROM pg_get_wal_records_info(:'wal_lsn1', :'wal_lsn2') + WHERE block_ref LIKE concat('%', :'sample_tbl_oid', '%') AND resource_manager = 'Heap'; + ok +---- + t +(1 row) + +-- =================================================================== +-- Tests for permissions +-- =================================================================== +CREATE ROLE regress_pg_walinspect; +SELECT has_function_privilege('regress_pg_walinspect', + 'pg_get_wal_record_info(pg_lsn)', 'EXECUTE'); -- no + has_function_privilege +------------------------ + f +(1 row) + +SELECT has_function_privilege('regress_pg_walinspect', + 'pg_get_wal_records_info(pg_lsn, pg_lsn) ', 'EXECUTE'); -- no + has_function_privilege +------------------------ + f +(1 row) + +SELECT has_function_privilege('regress_pg_walinspect', + 'pg_get_wal_stats(pg_lsn, pg_lsn, boolean) ', 'EXECUTE'); -- no + has_function_privilege +------------------------ + f +(1 row) + +-- Functions accessible by users with role pg_read_server_files +GRANT pg_read_server_files TO regress_pg_walinspect; +SELECT has_function_privilege('regress_pg_walinspect', + 'pg_get_wal_record_info(pg_lsn)', 'EXECUTE'); -- yes + has_function_privilege +------------------------ + t +(1 row) + +SELECT has_function_privilege('regress_pg_walinspect', + 'pg_get_wal_records_info(pg_lsn, pg_lsn) ', 'EXECUTE'); -- yes + has_function_privilege +------------------------ + t +(1 row) + +SELECT has_function_privilege('regress_pg_walinspect', + 'pg_get_wal_stats(pg_lsn, pg_lsn, boolean) ', 'EXECUTE'); -- yes + has_function_privilege +------------------------ + t +(1 row) + +REVOKE pg_read_server_files FROM regress_pg_walinspect; +-- Superuser can grant execute to other users +GRANT EXECUTE ON FUNCTION pg_get_wal_record_info(pg_lsn) + TO regress_pg_walinspect; +GRANT EXECUTE ON FUNCTION pg_get_wal_records_info(pg_lsn, pg_lsn) + TO regress_pg_walinspect; +GRANT EXECUTE ON FUNCTION pg_get_wal_stats(pg_lsn, pg_lsn, boolean) + TO regress_pg_walinspect; +SELECT has_function_privilege('regress_pg_walinspect', + 'pg_get_wal_record_info(pg_lsn)', 'EXECUTE'); -- yes + has_function_privilege +------------------------ + t +(1 row) + +SELECT has_function_privilege('regress_pg_walinspect', + 'pg_get_wal_records_info(pg_lsn, pg_lsn) ', 'EXECUTE'); -- yes + has_function_privilege +------------------------ + t +(1 row) + +SELECT has_function_privilege('regress_pg_walinspect', + 'pg_get_wal_stats(pg_lsn, pg_lsn, boolean) ', 'EXECUTE'); -- yes + has_function_privilege +------------------------ + t +(1 row) + +REVOKE EXECUTE ON FUNCTION pg_get_wal_record_info(pg_lsn) + FROM regress_pg_walinspect; +REVOKE EXECUTE ON FUNCTION pg_get_wal_records_info(pg_lsn, pg_lsn) + FROM regress_pg_walinspect; +REVOKE EXECUTE ON FUNCTION pg_get_wal_stats(pg_lsn, pg_lsn, boolean) + FROM regress_pg_walinspect; +-- =================================================================== +-- Clean up +-- =================================================================== +DROP ROLE regress_pg_walinspect; +DROP TABLE sample_tbl; diff --git a/contrib/pg_walinspect/sql/pg_walinspect.sql b/contrib/pg_walinspect/sql/pg_walinspect.sql new file mode 100644 index 0000000000..6e120c472b --- /dev/null +++ b/contrib/pg_walinspect/sql/pg_walinspect.sql @@ -0,0 +1,107 @@ +CREATE EXTENSION pg_walinspect; + +CREATE TABLE sample_tbl(col1 int, col2 int); + +SELECT pg_current_wal_lsn() AS wal_lsn1 \gset + +INSERT INTO sample_tbl SELECT * FROM generate_series(1, 2); + +SELECT pg_current_wal_lsn() AS wal_lsn2 \gset + +INSERT INTO sample_tbl SELECT * FROM generate_series(1, 2); + +-- =================================================================== +-- Tests for input validation +-- =================================================================== + +SELECT COUNT(*) >= 0 AS ok FROM pg_get_wal_records_info(:'wal_lsn2', :'wal_lsn1'); -- ERROR + +SELECT COUNT(*) >= 0 AS ok FROM pg_get_wal_stats(:'wal_lsn2', :'wal_lsn1'); -- ERROR + +-- =================================================================== +-- Tests for all function executions +-- =================================================================== + +SELECT COUNT(*) >= 0 AS ok FROM pg_get_wal_record_info(:'wal_lsn1'); + +SELECT COUNT(*) >= 0 AS ok FROM pg_get_wal_records_info(:'wal_lsn1', :'wal_lsn2'); + +SELECT COUNT(*) >= 0 AS ok FROM pg_get_wal_records_info_till_end_of_wal(:'wal_lsn1'); + +SELECT COUNT(*) >= 0 AS ok FROM pg_get_wal_stats(:'wal_lsn1', :'wal_lsn2'); + +SELECT COUNT(*) >= 0 AS ok FROM pg_get_wal_stats_till_end_of_wal(:'wal_lsn1'); + +-- =================================================================== +-- Tests for filtering out WAL records of a particular table +-- =================================================================== + +SELECT oid AS sample_tbl_oid FROM pg_class WHERE relname = 'sample_tbl' \gset + +SELECT COUNT(*) >= 1 AS ok FROM pg_get_wal_records_info(:'wal_lsn1', :'wal_lsn2') + WHERE block_ref LIKE concat('%', :'sample_tbl_oid', '%') AND resource_manager = 'Heap'; + +-- =================================================================== +-- Tests for permissions +-- =================================================================== +CREATE ROLE regress_pg_walinspect; + +SELECT has_function_privilege('regress_pg_walinspect', + 'pg_get_wal_record_info(pg_lsn)', 'EXECUTE'); -- no + +SELECT has_function_privilege('regress_pg_walinspect', + 'pg_get_wal_records_info(pg_lsn, pg_lsn) ', 'EXECUTE'); -- no + +SELECT has_function_privilege('regress_pg_walinspect', + 'pg_get_wal_stats(pg_lsn, pg_lsn, boolean) ', 'EXECUTE'); -- no + +-- Functions accessible by users with role pg_read_server_files + +GRANT pg_read_server_files TO regress_pg_walinspect; + +SELECT has_function_privilege('regress_pg_walinspect', + 'pg_get_wal_record_info(pg_lsn)', 'EXECUTE'); -- yes + +SELECT has_function_privilege('regress_pg_walinspect', + 'pg_get_wal_records_info(pg_lsn, pg_lsn) ', 'EXECUTE'); -- yes + +SELECT has_function_privilege('regress_pg_walinspect', + 'pg_get_wal_stats(pg_lsn, pg_lsn, boolean) ', 'EXECUTE'); -- yes + +REVOKE pg_read_server_files FROM regress_pg_walinspect; + +-- Superuser can grant execute to other users +GRANT EXECUTE ON FUNCTION pg_get_wal_record_info(pg_lsn) + TO regress_pg_walinspect; + +GRANT EXECUTE ON FUNCTION pg_get_wal_records_info(pg_lsn, pg_lsn) + TO regress_pg_walinspect; + +GRANT EXECUTE ON FUNCTION pg_get_wal_stats(pg_lsn, pg_lsn, boolean) + TO regress_pg_walinspect; + +SELECT has_function_privilege('regress_pg_walinspect', + 'pg_get_wal_record_info(pg_lsn)', 'EXECUTE'); -- yes + +SELECT has_function_privilege('regress_pg_walinspect', + 'pg_get_wal_records_info(pg_lsn, pg_lsn) ', 'EXECUTE'); -- yes + +SELECT has_function_privilege('regress_pg_walinspect', + 'pg_get_wal_stats(pg_lsn, pg_lsn, boolean) ', 'EXECUTE'); -- yes + +REVOKE EXECUTE ON FUNCTION pg_get_wal_record_info(pg_lsn) + FROM regress_pg_walinspect; + +REVOKE EXECUTE ON FUNCTION pg_get_wal_records_info(pg_lsn, pg_lsn) + FROM regress_pg_walinspect; + +REVOKE EXECUTE ON FUNCTION pg_get_wal_stats(pg_lsn, pg_lsn, boolean) + FROM regress_pg_walinspect; + +-- =================================================================== +-- Clean up +-- =================================================================== + +DROP ROLE regress_pg_walinspect; + +DROP TABLE sample_tbl; -- 2.25.1 [application/octet-stream] v17-0004-pg_walinspect-docs.patch (15.1K, ../../CALj2ACXC8jmF1d74Mb9Ca9sVsN4biVDTka3=x6XO22XDM-t1jw@mail.gmail.com/5-v17-0004-pg_walinspect-docs.patch) download | inline diff: From 766f3f862fbcc19fc6e961f82dca59ab8386149b Mon Sep 17 00:00:00 2001 From: Bharath Rupireddy <[email protected]> Date: Wed, 6 Apr 2022 08:00:06 +0000 Subject: [PATCH v17] pg_walinspect docs --- doc/src/sgml/contrib.sgml | 1 + doc/src/sgml/filelist.sgml | 1 + doc/src/sgml/pgwalinspect.sgml | 244 +++++++++++++++++++++++++++++++++ 3 files changed, 246 insertions(+) create mode 100644 doc/src/sgml/pgwalinspect.sgml diff --git a/doc/src/sgml/contrib.sgml b/doc/src/sgml/contrib.sgml index 1e42ce1a7f..4e7b87a42f 100644 --- a/doc/src/sgml/contrib.sgml +++ b/doc/src/sgml/contrib.sgml @@ -131,6 +131,7 @@ CREATE EXTENSION <replaceable>module_name</replaceable>; &pgsurgery; &pgtrgm; &pgvisibility; + &pgwalinspect; &postgres-fdw; &seg; &sepgsql; diff --git a/doc/src/sgml/filelist.sgml b/doc/src/sgml/filelist.sgml index fd853af01f..34c19c80f1 100644 --- a/doc/src/sgml/filelist.sgml +++ b/doc/src/sgml/filelist.sgml @@ -147,6 +147,7 @@ <!ENTITY pgsurgery SYSTEM "pgsurgery.sgml"> <!ENTITY pgtrgm SYSTEM "pgtrgm.sgml"> <!ENTITY pgvisibility SYSTEM "pgvisibility.sgml"> +<!ENTITY pgwalinspect SYSTEM "pgwalinspect.sgml"> <!ENTITY postgres-fdw SYSTEM "postgres-fdw.sgml"> <!ENTITY seg SYSTEM "seg.sgml"> <!ENTITY contrib-spi SYSTEM "contrib-spi.sgml"> diff --git a/doc/src/sgml/pgwalinspect.sgml b/doc/src/sgml/pgwalinspect.sgml new file mode 100644 index 0000000000..e344951f0d --- /dev/null +++ b/doc/src/sgml/pgwalinspect.sgml @@ -0,0 +1,244 @@ +<!-- doc/src/sgml/pgwalinspect.sgml --> + +<sect1 id="pgwalinspect" xreflabel="pg_walinspect"> + <title>pg_walinspect</title> + + <indexterm zone="pgwalinspect"> + <primary>pg_walinspect</primary> + </indexterm> + + <para> + The <filename>pg_walinspect</filename> module provides functions that allow + you to inspect the contents of write-ahead log of <productname>PostgreSQL</productname> + database cluster at a low level, which is useful for debugging or analytical + or reporting or educational purposes. + </para> + + <para> + All the functions of this module will provide the WAL information using the + current server's timeline ID. + </para> + + <para> + By default, use of these functions is restricted to superusers and members of + the <literal>pg_read_server_files</literal> role. Access may be granted by + superusers to others using <command>GRANT</command>. + </para> + + <sect2> + <title>General Functions</title> + + <variablelist> + <varlistentry> + <term> + <function> + pg_get_wal_record_info(in_lsn pg_lsn, + start_lsn OUT pg_lsn, + end_lsn OUT pg_lsn, + prev_lsn OUT pg_lsn, + xid OUT xid, + resource_manager OUT text, + record_type OUT text, + record_length OUT int4, + main_data_length OUT int4, + fpi_length OUT int4, + description OUT text, + block_ref OUT text) + </function> + </term> + + <listitem> + <para> + Gets WAL record information of a given LSN. If the given LSN isn't + containing a valid WAL record, it gives the information of the next + available valid WAL record. This function emits an error if a future (the + LSN database system doesn't know about) <replaceable>in_lsn</replaceable> + is specified. + </para> + </listitem> + </varlistentry> + + <varlistentry> + <term> + <function> + pg_get_wal_records_info(start_lsn pg_lsn, + end_lsn pg_lsn, + start_lsn OUT pg_lsn, + end_lsn OUT pg_lsn, + prev_lsn OUT pg_lsn, + xid OUT xid, + resource_manager OUT text, + record_type OUT text, + record_length OUT int4, + main_data_length OUT int4, + fpi_length OUT int4, + description OUT text, + block_ref OUT text) + returns setof record + </function> + </term> + + <listitem> + <para> + Gets information of all the valid WAL records between + <replaceable>start_lsn</replaceable> and <replaceable>end_lsn</replaceable>. + Returns one row per each valid WAL record. This function emits an error + if a future (the LSN database system doesn't know about) + <replaceable>start_lsn</replaceable> or <replaceable>end_lsn</replaceable> + is specified. For example, usage of the function is as follows: +<screen> +postgres=# select start_lsn, end_lsn, prev_lsn, xid, resource_manager, record_type, record_length, main_data_length, fpi_length, description from pg_get_wal_records_info('0/14FBA30', '0/15011D7'); + start_lsn | end_lsn | prev_lsn | xid | resource_manager | record_type | record_length | main_data_length | fpi_length | description +-----------+-----------+-----------+-----+------------------+---------------+---------------+------------------+------------+--------------------------------------------------------- + 0/14FBA30 | 0/14FBA67 | 0/14FB9F8 | 0 | Heap2 | PRUNE | 56 | 8 | 0 | latestRemovedXid 0 nredirected 0 ndead 1 + 0/14FBA68 | 0/14FBA9F | 0/14FBA30 | 0 | Standby | RUNNING_XACTS | 50 | 24 | 0 | nextXid 723 latestCompletedXid 722 oldestRunningXid 723 + 0/14FBAA0 | 0/14FBACF | 0/14FBA68 | 0 | Storage | CREATE | 42 | 16 | 0 | base/5/16390 + 0/14FBAD0 | 0/14FC117 | 0/14FBAA0 | 723 | Heap | INSERT | 1582 | 3 | 1528 | off 8 flags 0x01 + 0/14FC118 | 0/14FD487 | 0/14FBAD0 | 723 | Btree | INSERT_LEAF | 4973 | 2 | 4920 | off 244 + 0/14FD488 | 0/14FEFCF | 0/14FC118 | 723 | Btree | INSERT_LEAF | 6953 | 2 | 6900 | off 126 + 0/14FEFD0 | 0/14FF027 | 0/14FD488 | 723 | Heap2 | MULTI_INSERT | 85 | 6 | 0 | 1 tuples flags 0x02 + 0/14FF028 | 0/14FF06F | 0/14FEFD0 | 723 | Btree | INSERT_LEAF | 72 | 2 | 0 | off 155 + 0/14FF070 | 0/14FF0B7 | 0/14FF028 | 723 | Btree | INSERT_LEAF | 72 | 2 | 0 | off 134 + 0/14FF0B8 | 0/14FF18F | 0/14FF070 | 723 | Heap | INSERT | 211 | 3 | 0 | off 9 flags 0x00 + 0/14FF190 | 0/14FF1CF | 0/14FF0B8 | 723 | Btree | INSERT_LEAF | 64 | 2 | 0 | off 244 + 0/14FF1D0 | 0/15010B7 | 0/14FF190 | 723 | Btree | SPLIT_L | 7885 | 10 | 4136 | level 0, firstrightoff 120, newitemoff 47, postingoff 0 + 0/15010B8 | 0/150117F | 0/14FF1D0 | 723 | Btree | INSERT_UPPER | 197 | 2 | 136 | off 2 + 0/1501180 | 0/15011D7 | 0/15010B8 | 723 | Heap2 | MULTI_INSERT | 85 | 6 | 0 | 1 tuples flags 0x02 +(14 rows) +</screen> + </para> + </listitem> + </varlistentry> + + <varlistentry> + <term> + <function> + pg_get_wal_records_info_till_end_of_wal(start_lsn pg_lsn, + start_lsn OUT pg_lsn, + end_lsn OUT pg_lsn, + prev_lsn OUT pg_lsn, + xid OUT xid, + resource_manager OUT text, + record_type OUT text, + record_length OUT int4, + main_data_length OUT int4, + fpi_length OUT int4, + description OUT text, + block_ref OUT text) + returns setof record + </function> + </term> + + <listitem> + <para> + This function is same as <function>pg_get_wal_records_info()</function> + except that it gets information of all the valid WAL records from + <replaceable>start_lsn</replaceable> till end of WAL. + </para> + </listitem> + </varlistentry> + + <varlistentry> + <term> + <function> + pg_get_wal_stats(start_lsn pg_lsn, + end_lsn pg_lsn, + per_record boolean DEFAULT false, + "resource_manager/record_type" OUT text, + count OUT int8, + count_percentage OUT float4, + record_length OUT int8, + record_length_percentage OUT float4, + fpi_length OUT int8, + fpi_length_percentage OUT float4, + combined_size OUT int8, + combined_size_percentage OUT float4) + returns setof record + </function> + </term> + + <listitem> + <para> + Gets statistics of all the valid WAL records between + <replaceable>start_lsn</replaceable> and + <replaceable>end_lsn</replaceable>. By default, it returns one row per + <replaceable>resource_manager</replaceable> type. When + <replaceable>per_record</replaceable> is set to <literal>true</literal>, + it returns one row per <replaceable>record_type</replaceable>. This + function emits an error if a future (the LSN database system doesn't know + about) <replaceable>start_lsn</replaceable> or <replaceable>end_lsn</replaceable> + is specified. For example, usage of the function is as follows: +<screen> +postgres=# select * from pg_get_wal_stats('0/12FBA30', '0/15011D7') where count > 0; + resource_manager/record_type | count | count_percentage | record_size | record_size_percentage | fpi_size | fpi_size_percentage | combined_size | combined_size_percentage +------------------------------+-------+------------------+-------------+------------------------+----------+---------------------+---------------+-------------------------- + XLOG | 12 | 0.13002492 | 1024 | 2.3833392e-05 | 352 | 0.06136488 | 1376 | 3.2021846e-05 + Transaction | 188 | 2.0370572 | 62903 | 0.0014640546 | 0 | 0 | 62903 | 0.0014638591 + Storage | 13 | 0.14086033 | 546 | 1.2708038e-05 | 0 | 0 | 546 | 1.2706342e-05 + Database | 2 | 0.02167082 | 84 | 1.955083e-06 | 0 | 0 | 84 | 1.954822e-06 + Standby | 219 | 2.3729548 | 15830 | 0.00036844003 | 0 | 0 | 15830 | 0.00036839084 + Heap2 | 1905 | 20.641457 | 384619 | 0.0089519285 | 364472 | 63.53915 | 749091 | 0.017432613 + Heap | 1319 | 14.291906 | 621997 | 0.014476853 | 145232 | 25.318592 | 767229 | 0.017854715 + Btree | 5571 | 60.36407 | 4295405999 | 99.9747 | 63562 | 11.0808935 | 4295469561 | 99.96284 +(8 rows) +</screen> + +With <replaceable>per_record</replaceable> passed as <literal>true</literal>: + +<screen> +postgres=# select * from pg_get_wal_stats('0/14FBA30', '0/15011D7', true) where count > 0; + resource_manager/record_type | count | count_percentage | record_size | record_size_percentage | fpi_size | fpi_size_percentage | combined_size | combined_size_percentage +------------------------------+-------+------------------+-------------+------------------------+----------+---------------------+---------------+-------------------------- + Storage/CREATE | 1 | 7.142857 | 42 | 0.8922881 | 0 | 0 | 42 | 0.18811305 + Standby/RUNNING_XACTS | 1 | 7.142857 | 50 | 1.0622478 | 0 | 0 | 50 | 0.2239441 + Heap2/PRUNE | 1 | 7.142857 | 56 | 1.1897174 | 0 | 0 | 56 | 0.2508174 + Heap2/MULTI_INSERT | 2 | 14.285714 | 170 | 3.6116421 | 0 | 0 | 170 | 0.76140994 + Heap/INSERT | 2 | 14.285714 | 265 | 5.629913 | 1528 | 8.671964 | 1793 | 8.030636 + Btree/INSERT_LEAF | 5 | 35.714287 | 314 | 6.6709156 | 11820 | 67.08286 | 12134 | 54.346756 + Btree/INSERT_UPPER | 1 | 7.142857 | 61 | 1.2959422 | 136 | 0.77185017 | 197 | 0.8823398 + Btree/SPLIT_L | 1 | 7.142857 | 3749 | 79.64733 | 4136 | 23.473326 | 7885 | 35.315987 +(8 rows) +</screen> + </para> + </listitem> + </varlistentry> + + <varlistentry> + <term> + <function> + pg_get_wal_stats_till_end_of_wal(start_lsn pg_lsn, + per_record boolean DEFAULT false, + "resource_manager/record_type" OUT text, + count OUT int8, + count_percentage OUT float4, + record_length OUT int8, + record_length_percentage OUT float4, + fpi_length OUT int8, + fpi_length_percentage OUT float4, + combined_size OUT int8, + combined_size_percentage OUT float4) + returns setof record + </function> + </term> + + <listitem> + <para> + This function is same as <function>pg_get_wal_stats()</function> except + that it gets statistics of all the valid WAL records from + <replaceable>start_lsn</replaceable> till end of WAL. + </para> + </listitem> + </varlistentry> + + </variablelist> + </sect2> + + <sect2> + <title>Author</title> + + <para> + Bharath Rupireddy <email>[email protected]</email> + </para> + </sect2> + +</sect1> -- 2.25.1 ^ permalink raw reply [nested|flat] 9+ messages in thread
* Re: pg_walinspect - a new extension to get raw WAL data and WAL stats 2022-04-04 03:45 Re: pg_walinspect - a new extension to get raw WAL data and WAL stats Bharath Rupireddy <[email protected]> 2022-04-06 05:02 ` Re: pg_walinspect - a new extension to get raw WAL data and WAL stats Jeff Davis <[email protected]> 2022-04-06 08:45 ` Re: pg_walinspect - a new extension to get raw WAL data and WAL stats Bharath Rupireddy <[email protected]> @ 2022-04-07 10:05 ` Bharath Rupireddy <[email protected]> 2022-04-11 10:51 ` Re: pg_walinspect - a new extension to get raw WAL data and WAL stats Amit Kapila <[email protected]> 0 siblings, 1 reply; 9+ messages in thread From: Bharath Rupireddy @ 2022-04-07 10:05 UTC (permalink / raw) To: Jeff Davis <[email protected]>; +Cc: RKN Sai Krishna <[email protected]>; Andres Freund <[email protected]>; Kyotaro Horiguchi <[email protected]>; Ashutosh Sharma <[email protected]>; Stephen Frost <[email protected]>; Robert Haas <[email protected]>; Andrew Dunstan <[email protected]>; Greg Stark <[email protected]>; Jeremy Schneider <[email protected]>; Bruce Momjian <[email protected]>; PostgreSQL Hackers <[email protected]>; SATYANARAYANA NARLAPURAM <[email protected]>; [email protected]; [email protected] On Wed, Apr 6, 2022 at 2:15 PM Bharath Rupireddy <[email protected]> wrote: > > Attaching v17 patch-set with the above review comments addressed. > Please have a look at it. Had to rebase because of 5c279a6d350 (Custom WAL Resource Managers.). Please find v18 patch-set. Regards, Bharath Rupireddy. Attachments: [application/x-patch] v18-0001-Refactor-pg_waldump-code.patch (18.3K, ../../CALj2ACV7cXcCO-Ecyc9+_f71WeN-_HNqk=HJ7Ymvp3dXqN_s1A@mail.gmail.com/2-v18-0001-Refactor-pg_waldump-code.patch) download | inline diff: From eb9e3464340a4e09236b9f50e596c9a1360b0886 Mon Sep 17 00:00:00 2001 From: Bharath Rupireddy <[email protected]> Date: Thu, 7 Apr 2022 08:36:01 +0000 Subject: [PATCH v18] Refactor pg_waldump code This patch puts some generic chunks of pg_waldump's code into separate reusable functions in xlogdesc.c and xlogstats.c, a new file along xlogstats.h introduced for placing WAL stats and structures. This way, other modules can reuse these common functions. --- src/backend/access/rmgrdesc/xlogdesc.c | 125 ++++++++++++++++ src/backend/access/transam/Makefile | 1 + src/backend/access/transam/xlogstats.c | 93 ++++++++++++ src/bin/pg_waldump/.gitignore | 1 + src/bin/pg_waldump/Makefile | 8 +- src/bin/pg_waldump/pg_waldump.c | 199 ++----------------------- src/include/access/xlog_internal.h | 5 + src/include/access/xlogstats.h | 40 +++++ 8 files changed, 282 insertions(+), 190 deletions(-) create mode 100644 src/backend/access/transam/xlogstats.c create mode 100644 src/include/access/xlogstats.h diff --git a/src/backend/access/rmgrdesc/xlogdesc.c b/src/backend/access/rmgrdesc/xlogdesc.c index e7452af679..429e5dcd5b 100644 --- a/src/backend/access/rmgrdesc/xlogdesc.c +++ b/src/backend/access/rmgrdesc/xlogdesc.c @@ -200,3 +200,128 @@ xlog_identify(uint8 info) return id; } + +/* + * Returns a string giving information about all the blocks in an + * XLogRecord. + */ +void +XLogRecGetBlockRefInfo(XLogReaderState *record, char *delimiter, + uint32 *fpi_len, bool detailed_format, + StringInfo buf) +{ + int block_id; + + Assert(record != NULL); + + if (detailed_format && delimiter != NULL) + appendStringInfoChar(buf, '\n'); + + for (block_id = 0; block_id <= XLogRecMaxBlockId(record); block_id++) + { + RelFileNode rnode = {InvalidOid, InvalidOid, InvalidOid}; + ForkNumber forknum = InvalidForkNumber; + BlockNumber blk = InvalidBlockNumber; + + if (!XLogRecHasBlockRef(record, block_id)) + continue; + + XLogRecGetBlockTag(record, block_id, &rnode, &forknum, &blk); + + if (detailed_format) + { + /* Get block references in detailed format. */ + + appendStringInfo(buf, + "\tblkref #%d: rel %u/%u/%u fork %s blk %u", + block_id, + rnode.spcNode, rnode.dbNode, rnode.relNode, + forkNames[forknum], + blk); + + if (XLogRecHasBlockImage(record, block_id)) + { + uint8 bimg_info = XLogRecGetBlock(record, block_id)->bimg_info; + + /* Calculate the amount of FPI data in the record. */ + if (fpi_len) + *fpi_len += XLogRecGetBlock(record, block_id)->bimg_len; + + if (BKPIMAGE_COMPRESSED(bimg_info)) + { + const char *method; + + if ((bimg_info & BKPIMAGE_COMPRESS_PGLZ) != 0) + method = "pglz"; + else if ((bimg_info & BKPIMAGE_COMPRESS_LZ4) != 0) + method = "lz4"; + else if ((bimg_info & BKPIMAGE_COMPRESS_ZSTD) != 0) + method = "zstd"; + else + method = "unknown"; + + appendStringInfo(buf, + " (FPW%s); hole: offset: %u, length: %u, " + "compression saved: %u, method: %s", + XLogRecBlockImageApply(record, block_id) ? + "" : " for WAL verification", + XLogRecGetBlock(record, block_id)->hole_offset, + XLogRecGetBlock(record, block_id)->hole_length, + BLCKSZ - + XLogRecGetBlock(record, block_id)->hole_length - + XLogRecGetBlock(record, block_id)->bimg_len, + method); + } + else + { + appendStringInfo(buf, + " (FPW%s); hole: offset: %u, length: %u", + XLogRecBlockImageApply(record, block_id) ? + "" : " for WAL verification", + XLogRecGetBlock(record, block_id)->hole_offset, + XLogRecGetBlock(record, block_id)->hole_length); + } + } + } + else + { + /* Get block references in short format. */ + + if (forknum != MAIN_FORKNUM) + { + appendStringInfo(buf, + ", blkref #%d: rel %u/%u/%u fork %s blk %u", + block_id, + rnode.spcNode, rnode.dbNode, rnode.relNode, + forkNames[forknum], + blk); + } + else + { + appendStringInfo(buf, + ", blkref #%d: rel %u/%u/%u blk %u", + block_id, + rnode.spcNode, rnode.dbNode, rnode.relNode, + blk); + } + + if (XLogRecHasBlockImage(record, block_id)) + { + /* Calculate the amount of FPI data in the record. */ + if (fpi_len) + *fpi_len += XLogRecGetBlock(record, block_id)->bimg_len; + + if (XLogRecBlockImageApply(record, block_id)) + appendStringInfo(buf, " FPW"); + else + appendStringInfo(buf, " FPW for WAL verification"); + } + } + + if (detailed_format && delimiter != NULL) + appendStringInfoChar(buf, '\n'); + } + + if (!detailed_format && delimiter != NULL) + appendStringInfoChar(buf, '\n'); +} diff --git a/src/backend/access/transam/Makefile b/src/backend/access/transam/Makefile index 8c17c88dfc..3e5444a6f7 100644 --- a/src/backend/access/transam/Makefile +++ b/src/backend/access/transam/Makefile @@ -34,6 +34,7 @@ OBJS = \ xlogprefetcher.o \ xlogreader.o \ xlogrecovery.o \ + xlogstats.o \ xlogutils.o include $(top_srcdir)/src/backend/common.mk diff --git a/src/backend/access/transam/xlogstats.c b/src/backend/access/transam/xlogstats.c new file mode 100644 index 0000000000..aff3069ecb --- /dev/null +++ b/src/backend/access/transam/xlogstats.c @@ -0,0 +1,93 @@ +/*------------------------------------------------------------------------- + * + * xlogstats.c + * Functions for WAL Statitstics + * + * Copyright (c) 2022, PostgreSQL Global Development Group + * + * IDENTIFICATION + * src/backend/access/transam/xlogstats.c + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include "access/xlogreader.h" +#include "access/xlogstats.h" + +/* + * Calculate the size of a record, split into !FPI and FPI parts. + */ +void +XLogRecGetLen(XLogReaderState *record, uint32 *rec_len, + uint32 *fpi_len) +{ + int block_id; + + /* + * Calculate the amount of FPI data in the record. + * + * XXX: We peek into xlogreader's private decoded backup blocks for the + * bimg_len indicating the length of FPI data. + */ + *fpi_len = 0; + for (block_id = 0; block_id <= XLogRecMaxBlockId(record); block_id++) + { + if (XLogRecHasBlockImage(record, block_id)) + *fpi_len += XLogRecGetBlock(record, block_id)->bimg_len; + } + + /* + * Calculate the length of the record as the total length - the length of + * all the block images. + */ + *rec_len = XLogRecGetTotalLen(record) - *fpi_len; +} + +/* + * Store per-rmgr and per-record statistics for a given record. + */ +void +XLogRecStoreStats(XLogStats *stats, XLogReaderState *record) +{ + RmgrId rmid; + uint8 recid; + uint32 rec_len; + uint32 fpi_len; + + Assert(stats != NULL && record != NULL); + + stats->count++; + + rmid = XLogRecGetRmid(record); + + XLogRecGetLen(record, &rec_len, &fpi_len); + + /* Update per-rmgr statistics */ + + stats->rmgr_stats[rmid].count++; + stats->rmgr_stats[rmid].rec_len += rec_len; + stats->rmgr_stats[rmid].fpi_len += fpi_len; + + /* + * Update per-record statistics, where the record is identified by a + * combination of the RmgrId and the four bits of the xl_info field that + * are the rmgr's domain (resulting in sixteen possible entries per + * RmgrId). + */ + + recid = XLogRecGetInfo(record) >> 4; + + /* + * XACT records need to be handled differently. Those records use the + * first bit of those four bits for an optional flag variable and the + * following three bits for the opcode. We filter opcode out of xl_info + * and use it as the identifier of the record. + */ + if (rmid == RM_XACT_ID) + recid &= 0x07; + + stats->record_stats[rmid][recid].count++; + stats->record_stats[rmid][recid].rec_len += rec_len; + stats->record_stats[rmid][recid].fpi_len += fpi_len; +} diff --git a/src/bin/pg_waldump/.gitignore b/src/bin/pg_waldump/.gitignore index 3be00a8b61..dabb6e34b6 100644 --- a/src/bin/pg_waldump/.gitignore +++ b/src/bin/pg_waldump/.gitignore @@ -23,6 +23,7 @@ /xactdesc.c /xlogdesc.c /xlogreader.c +/xlogstat.c # Generated by test suite /tmp_check/ diff --git a/src/bin/pg_waldump/Makefile b/src/bin/pg_waldump/Makefile index 9f333d0c8a..d6459e17c7 100644 --- a/src/bin/pg_waldump/Makefile +++ b/src/bin/pg_waldump/Makefile @@ -13,7 +13,8 @@ OBJS = \ compat.o \ pg_waldump.o \ rmgrdesc.o \ - xlogreader.o + xlogreader.o \ + xlogstats.o override CPPFLAGS := -DFRONTEND $(CPPFLAGS) @@ -29,6 +30,9 @@ pg_waldump: $(OBJS) | submake-libpgport xlogreader.c: % : $(top_srcdir)/src/backend/access/transam/% rm -f $@ && $(LN_S) $< . +xlogstats.c: % : $(top_srcdir)/src/backend/access/transam/% + rm -f $@ && $(LN_S) $< . + $(RMGRDESCSOURCES): % : $(top_srcdir)/src/backend/access/rmgrdesc/% rm -f $@ && $(LN_S) $< . @@ -42,7 +46,7 @@ uninstall: rm -f '$(DESTDIR)$(bindir)/pg_waldump$(X)' clean distclean maintainer-clean: - rm -f pg_waldump$(X) $(OBJS) $(RMGRDESCSOURCES) xlogreader.c + rm -f pg_waldump$(X) $(OBJS) $(RMGRDESCSOURCES) xlogreader.c xlogstats.c rm -rf tmp_check check: diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c index 30ca7684bd..a0ab1912f4 100644 --- a/src/bin/pg_waldump/pg_waldump.c +++ b/src/bin/pg_waldump/pg_waldump.c @@ -21,6 +21,7 @@ #include "access/xlog_internal.h" #include "access/xlogreader.h" #include "access/xlogrecord.h" +#include "access/xlogstats.h" #include "common/fe_memutils.h" #include "common/logging.h" #include "getopt_long.h" @@ -66,24 +67,6 @@ typedef struct XLogDumpConfig bool filter_by_fpw; } XLogDumpConfig; -typedef struct Stats -{ - uint64 count; - uint64 rec_len; - uint64 fpi_len; -} Stats; - -#define MAX_XLINFO_TYPES 16 - -typedef struct XLogDumpStats -{ - uint64 count; - XLogRecPtr startptr; - XLogRecPtr endptr; - Stats rmgr_stats[RM_MAX_ID + 1]; - Stats record_stats[RM_MAX_ID + 1][MAX_XLINFO_TYPES]; -} XLogDumpStats; - #define fatal_error(...) do { pg_log_fatal(__VA_ARGS__); exit(EXIT_FAILURE); } while(0) /* @@ -453,81 +436,6 @@ XLogRecordHasFPW(XLogReaderState *record) return false; } -/* - * Calculate the size of a record, split into !FPI and FPI parts. - */ -static void -XLogDumpRecordLen(XLogReaderState *record, uint32 *rec_len, uint32 *fpi_len) -{ - int block_id; - - /* - * Calculate the amount of FPI data in the record. - * - * XXX: We peek into xlogreader's private decoded backup blocks for the - * bimg_len indicating the length of FPI data. - */ - *fpi_len = 0; - for (block_id = 0; block_id <= XLogRecMaxBlockId(record); block_id++) - { - if (XLogRecHasBlockImage(record, block_id)) - *fpi_len += XLogRecGetBlock(record, block_id)->bimg_len; - } - - /* - * Calculate the length of the record as the total length - the length of - * all the block images. - */ - *rec_len = XLogRecGetTotalLen(record) - *fpi_len; -} - -/* - * Store per-rmgr and per-record statistics for a given record. - */ -static void -XLogDumpCountRecord(XLogDumpConfig *config, XLogDumpStats *stats, - XLogReaderState *record) -{ - RmgrId rmid; - uint8 recid; - uint32 rec_len; - uint32 fpi_len; - - stats->count++; - - rmid = XLogRecGetRmid(record); - - XLogDumpRecordLen(record, &rec_len, &fpi_len); - - /* Update per-rmgr statistics */ - - stats->rmgr_stats[rmid].count++; - stats->rmgr_stats[rmid].rec_len += rec_len; - stats->rmgr_stats[rmid].fpi_len += fpi_len; - - /* - * Update per-record statistics, where the record is identified by a - * combination of the RmgrId and the four bits of the xl_info field that - * are the rmgr's domain (resulting in sixteen possible entries per - * RmgrId). - */ - - recid = XLogRecGetInfo(record) >> 4; - - /* - * XACT records need to be handled differently. Those records use the - * first bit of those four bits for an optional flag variable and the - * following three bits for the opcode. We filter opcode out of xl_info - * and use it as the identifier of the record. - */ - if (rmid == RM_XACT_ID) - recid &= 0x07; - - stats->record_stats[rmid][recid].count++; - stats->record_stats[rmid][recid].rec_len += rec_len; - stats->record_stats[rmid][recid].fpi_len += fpi_len; -} - /* * Print a record to stdout */ @@ -538,15 +446,12 @@ XLogDumpDisplayRecord(XLogDumpConfig *config, XLogReaderState *record) const RmgrDescData *desc = GetRmgrDesc(XLogRecGetRmid(record)); uint32 rec_len; uint32 fpi_len; - RelFileNode rnode; - ForkNumber forknum; - BlockNumber blk; - int block_id; uint8 info = XLogRecGetInfo(record); XLogRecPtr xl_prev = XLogRecGetPrev(record); StringInfoData s; + char delim = {'\n'}; - XLogDumpRecordLen(record, &rec_len, &fpi_len); + XLogRecGetLen(record, &rec_len, &fpi_len); printf("rmgr: %-11s len (rec/tot): %6u/%6u, tx: %10u, lsn: %X/%08X, prev %X/%08X, ", desc->rm_name, @@ -564,93 +469,11 @@ XLogDumpDisplayRecord(XLogDumpConfig *config, XLogReaderState *record) initStringInfo(&s); desc->rm_desc(&s, record); printf("%s", s.data); - pfree(s.data); - if (!config->bkp_details) - { - /* print block references (short format) */ - for (block_id = 0; block_id <= XLogRecMaxBlockId(record); block_id++) - { - if (!XLogRecHasBlockRef(record, block_id)) - continue; - - XLogRecGetBlockTag(record, block_id, &rnode, &forknum, &blk); - if (forknum != MAIN_FORKNUM) - printf(", blkref #%d: rel %u/%u/%u fork %s blk %u", - block_id, - rnode.spcNode, rnode.dbNode, rnode.relNode, - forkNames[forknum], - blk); - else - printf(", blkref #%d: rel %u/%u/%u blk %u", - block_id, - rnode.spcNode, rnode.dbNode, rnode.relNode, - blk); - if (XLogRecHasBlockImage(record, block_id)) - { - if (XLogRecBlockImageApply(record, block_id)) - printf(" FPW"); - else - printf(" FPW for WAL verification"); - } - } - putchar('\n'); - } - else - { - /* print block references (detailed format) */ - putchar('\n'); - for (block_id = 0; block_id <= XLogRecMaxBlockId(record); block_id++) - { - if (!XLogRecHasBlockRef(record, block_id)) - continue; - - XLogRecGetBlockTag(record, block_id, &rnode, &forknum, &blk); - printf("\tblkref #%d: rel %u/%u/%u fork %s blk %u", - block_id, - rnode.spcNode, rnode.dbNode, rnode.relNode, - forkNames[forknum], - blk); - if (XLogRecHasBlockImage(record, block_id)) - { - uint8 bimg_info = XLogRecGetBlock(record, block_id)->bimg_info; - - if (BKPIMAGE_COMPRESSED(bimg_info)) - { - const char *method; - - if ((bimg_info & BKPIMAGE_COMPRESS_PGLZ) != 0) - method = "pglz"; - else if ((bimg_info & BKPIMAGE_COMPRESS_LZ4) != 0) - method = "lz4"; - else if ((bimg_info & BKPIMAGE_COMPRESS_ZSTD) != 0) - method = "zstd"; - else - method = "unknown"; - - printf(" (FPW%s); hole: offset: %u, length: %u, " - "compression saved: %u, method: %s", - XLogRecBlockImageApply(record, block_id) ? - "" : " for WAL verification", - XLogRecGetBlock(record, block_id)->hole_offset, - XLogRecGetBlock(record, block_id)->hole_length, - BLCKSZ - - XLogRecGetBlock(record, block_id)->hole_length - - XLogRecGetBlock(record, block_id)->bimg_len, - method); - } - else - { - printf(" (FPW%s); hole: offset: %u, length: %u", - XLogRecBlockImageApply(record, block_id) ? - "" : " for WAL verification", - XLogRecGetBlock(record, block_id)->hole_offset, - XLogRecGetBlock(record, block_id)->hole_length); - } - } - putchar('\n'); - } - } + resetStringInfo(&s); + XLogRecGetBlockRefInfo(record, &delim, NULL, config->bkp_details, &s); + printf("%s", s.data); + pfree(s.data); } /* @@ -698,7 +521,7 @@ XLogDumpStatsRow(const char *name, * Display summary statistics about the records seen so far. */ static void -XLogDumpDisplayStats(XLogDumpConfig *config, XLogDumpStats *stats) +XLogDumpDisplayStats(XLogDumpConfig *config, XLogStats *stats) { int ri, rj; @@ -867,7 +690,7 @@ main(int argc, char **argv) XLogReaderState *xlogreader_state; XLogDumpPrivate private; XLogDumpConfig config; - XLogDumpStats stats; + XLogStats stats; XLogRecord *record; XLogRecPtr first_record; char *waldir = NULL; @@ -921,7 +744,7 @@ main(int argc, char **argv) memset(&private, 0, sizeof(XLogDumpPrivate)); memset(&config, 0, sizeof(XLogDumpConfig)); - memset(&stats, 0, sizeof(XLogDumpStats)); + memset(&stats, 0, sizeof(XLogStats)); private.timeline = 1; private.startptr = InvalidXLogRecPtr; @@ -1319,7 +1142,7 @@ main(int argc, char **argv) { if (config.stats == true) { - XLogDumpCountRecord(&config, &stats, xlogreader_state); + XLogRecStoreStats(&stats, xlogreader_state); stats.endptr = xlogreader_state->EndRecPtr; } else diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index f69ea2355d..d0206f7c74 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -348,6 +348,11 @@ extern XLogRecPtr RequestXLogSwitch(bool mark_unimportant); extern void GetOldestRestartPoint(XLogRecPtr *oldrecptr, TimeLineID *oldtli); +extern void XLogRecGetBlockRefInfo(XLogReaderState *record, + char *delimiter, uint32 *fpi_len, + bool detailed_format, + StringInfo blk_ref); + /* * Exported for the functions in timeline.c and xlogarchive.c. Only valid * in the startup process. diff --git a/src/include/access/xlogstats.h b/src/include/access/xlogstats.h new file mode 100644 index 0000000000..227d59ce17 --- /dev/null +++ b/src/include/access/xlogstats.h @@ -0,0 +1,40 @@ +/*------------------------------------------------------------------------- + * + * xlogstats.h + * Definitions for WAL Statitstics + * + * Copyright (c) 2022, PostgreSQL Global Development Group + * + * IDENTIFICATION + * src/include/access/xlogstats.h + * + *------------------------------------------------------------------------- + */ +#ifndef XLOGSTATS_H +#define XLOGSTATS_H + +#define MAX_XLINFO_TYPES 16 + +typedef struct XLogRecStats +{ + uint64 count; + uint64 rec_len; + uint64 fpi_len; +} XLogRecStats; + +typedef struct XLogStats +{ + uint64 count; +#ifdef FRONTEND + XLogRecPtr startptr; + XLogRecPtr endptr; +#endif + XLogRecStats rmgr_stats[RM_MAX_ID + 1]; + XLogRecStats record_stats[RM_MAX_ID + 1][MAX_XLINFO_TYPES]; +} XLogStats; + +extern void XLogRecGetLen(XLogReaderState *record, uint32 *rec_len, + uint32 *fpi_len); +extern void XLogRecStoreStats(XLogStats *stats, XLogReaderState *record); + +#endif /* XLOGSTATS_H */ -- 2.25.1 [application/x-patch] v18-0002-pg_walinspect.patch (29.9K, ../../CALj2ACV7cXcCO-Ecyc9+_f71WeN-_HNqk=HJ7Ymvp3dXqN_s1A@mail.gmail.com/3-v18-0002-pg_walinspect.patch) download | inline diff: From 827c8b6f4300acb99b779cec41ecc0411897bda8 Mon Sep 17 00:00:00 2001 From: Bharath Rupireddy <[email protected]> Date: Thu, 7 Apr 2022 09:15:59 +0000 Subject: [PATCH v18] pg_walinspect --- contrib/Makefile | 1 + contrib/pg_walinspect/.gitignore | 4 + contrib/pg_walinspect/Makefile | 26 + contrib/pg_walinspect/pg_walinspect--1.0.sql | 118 ++++ contrib/pg_walinspect/pg_walinspect.c | 650 +++++++++++++++++++ contrib/pg_walinspect/pg_walinspect.control | 5 + src/backend/access/transam/xlogreader.c | 13 +- src/backend/access/transam/xlogutils.c | 33 + src/bin/pg_waldump/pg_waldump.c | 5 + src/include/access/xlog.h | 2 +- src/include/access/xlog_internal.h | 2 +- src/include/access/xlogreader.h | 2 - src/include/access/xlogutils.h | 4 + 13 files changed, 854 insertions(+), 11 deletions(-) create mode 100644 contrib/pg_walinspect/.gitignore create mode 100644 contrib/pg_walinspect/Makefile create mode 100644 contrib/pg_walinspect/pg_walinspect--1.0.sql create mode 100644 contrib/pg_walinspect/pg_walinspect.c create mode 100644 contrib/pg_walinspect/pg_walinspect.control diff --git a/contrib/Makefile b/contrib/Makefile index 332b486ecc..bbf220407b 100644 --- a/contrib/Makefile +++ b/contrib/Makefile @@ -41,6 +41,7 @@ SUBDIRS = \ pgrowlocks \ pgstattuple \ pg_visibility \ + pg_walinspect \ postgres_fdw \ seg \ spi \ diff --git a/contrib/pg_walinspect/.gitignore b/contrib/pg_walinspect/.gitignore new file mode 100644 index 0000000000..5dcb3ff972 --- /dev/null +++ b/contrib/pg_walinspect/.gitignore @@ -0,0 +1,4 @@ +# Generated subdirectories +/log/ +/results/ +/tmp_check/ diff --git a/contrib/pg_walinspect/Makefile b/contrib/pg_walinspect/Makefile new file mode 100644 index 0000000000..c92a97447f --- /dev/null +++ b/contrib/pg_walinspect/Makefile @@ -0,0 +1,26 @@ +# contrib/pg_walinspect/Makefile + +MODULE_big = pg_walinspect +OBJS = \ + $(WIN32RES) \ + pg_walinspect.o +PGFILEDESC = "pg_walinspect - functions to inspect contents of PostgreSQL Write-Ahead Log" + +PG_CPPFLAGS = -I$(libpq_srcdir) +SHLIB_LINK_INTERNAL = $(libpq) + +EXTENSION = pg_walinspect +DATA = pg_walinspect--1.0.sql + +REGRESS = pg_walinspect + +ifdef USE_PGXS +PG_CONFIG = pg_config +PGXS := $(shell $(PG_CONFIG) --pgxs) +include $(PGXS) +else +subdir = contrib/pg_walinspect +top_builddir = ../.. +include $(top_builddir)/src/Makefile.global +include $(top_srcdir)/contrib/contrib-global.mk +endif diff --git a/contrib/pg_walinspect/pg_walinspect--1.0.sql b/contrib/pg_walinspect/pg_walinspect--1.0.sql new file mode 100644 index 0000000000..aae6456c18 --- /dev/null +++ b/contrib/pg_walinspect/pg_walinspect--1.0.sql @@ -0,0 +1,118 @@ +/* contrib/pg_walinspect/pg_walinspect--1.0.sql */ + +-- complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "CREATE EXTENSION pg_walinspect" to load this file. \quit + +-- +-- pg_get_wal_record_info() +-- +CREATE FUNCTION pg_get_wal_record_info(IN in_lsn pg_lsn, + OUT start_lsn pg_lsn, + OUT end_lsn pg_lsn, + OUT prev_lsn pg_lsn, + OUT xid xid, + OUT resource_manager text, + OUT record_type text, + OUT record_length int4, + OUT main_data_length int4, + OUT fpi_length int4, + OUT description text, + OUT block_ref text +) +AS 'MODULE_PATHNAME', 'pg_get_wal_record_info' +LANGUAGE C STRICT PARALLEL SAFE; + +REVOKE EXECUTE ON FUNCTION pg_get_wal_record_info(pg_lsn) FROM PUBLIC; +GRANT EXECUTE ON FUNCTION pg_get_wal_record_info(pg_lsn) TO pg_read_server_files; + +-- +-- pg_get_wal_records_info() +-- +CREATE FUNCTION pg_get_wal_records_info(IN start_lsn pg_lsn, + IN end_lsn pg_lsn, + OUT start_lsn pg_lsn, + OUT end_lsn pg_lsn, + OUT prev_lsn pg_lsn, + OUT xid xid, + OUT resource_manager text, + OUT record_type text, + OUT record_length int4, + OUT main_data_length int4, + OUT fpi_length int4, + OUT description text, + OUT block_ref text +) +RETURNS SETOF record +AS 'MODULE_PATHNAME', 'pg_get_wal_records_info' +LANGUAGE C STRICT PARALLEL SAFE; + +REVOKE EXECUTE ON FUNCTION pg_get_wal_records_info(pg_lsn, pg_lsn) FROM PUBLIC; +GRANT EXECUTE ON FUNCTION pg_get_wal_records_info(pg_lsn, pg_lsn) TO pg_read_server_files; + +-- +-- pg_get_wal_records_info_till_end_of_wal() +-- +CREATE FUNCTION pg_get_wal_records_info_till_end_of_wal(IN start_lsn pg_lsn, + OUT start_lsn pg_lsn, + OUT end_lsn pg_lsn, + OUT prev_lsn pg_lsn, + OUT xid xid, + OUT resource_manager text, + OUT record_type text, + OUT record_length int4, + OUT main_data_length int4, + OUT fpi_length int4, + OUT description text, + OUT block_ref text +) +RETURNS SETOF record +AS 'MODULE_PATHNAME', 'pg_get_wal_records_info_till_end_of_wal' +LANGUAGE C STRICT PARALLEL SAFE; + +REVOKE EXECUTE ON FUNCTION pg_get_wal_records_info_till_end_of_wal(pg_lsn) FROM PUBLIC; +GRANT EXECUTE ON FUNCTION pg_get_wal_records_info_till_end_of_wal(pg_lsn) TO pg_read_server_files; + +-- +-- pg_get_wal_stats() +-- +CREATE FUNCTION pg_get_wal_stats(IN start_lsn pg_lsn, + IN end_lsn pg_lsn, + IN per_record boolean DEFAULT false, + OUT "resource_manager/record_type" text, + OUT count int8, + OUT count_percentage float4, + OUT record_size int8, + OUT record_size_percentage float4, + OUT fpi_size int8, + OUT fpi_size_percentage float4, + OUT combined_size int8, + OUT combined_size_percentage float4 +) +RETURNS SETOF record +AS 'MODULE_PATHNAME', 'pg_get_wal_stats' +LANGUAGE C STRICT PARALLEL SAFE; + +REVOKE EXECUTE ON FUNCTION pg_get_wal_stats(pg_lsn, pg_lsn, boolean) FROM PUBLIC; +GRANT EXECUTE ON FUNCTION pg_get_wal_stats(pg_lsn, pg_lsn, boolean) TO pg_read_server_files; + +-- +-- pg_get_wal_stats_till_end_of_wal() +-- +CREATE FUNCTION pg_get_wal_stats_till_end_of_wal(IN start_lsn pg_lsn, + IN per_record boolean DEFAULT false, + OUT "resource_manager/record_type" text, + OUT count int8, + OUT count_percentage float4, + OUT record_size int8, + OUT record_size_percentage float4, + OUT fpi_size int8, + OUT fpi_size_percentage float4, + OUT combined_size int8, + OUT combined_size_percentage float4 +) +RETURNS SETOF record +AS 'MODULE_PATHNAME', 'pg_get_wal_stats_till_end_of_wal' +LANGUAGE C STRICT PARALLEL SAFE; + +REVOKE EXECUTE ON FUNCTION pg_get_wal_stats_till_end_of_wal(pg_lsn, boolean) FROM PUBLIC; +GRANT EXECUTE ON FUNCTION pg_get_wal_stats_till_end_of_wal(pg_lsn, boolean) TO pg_read_server_files; diff --git a/contrib/pg_walinspect/pg_walinspect.c b/contrib/pg_walinspect/pg_walinspect.c new file mode 100644 index 0000000000..6cb0a6df40 --- /dev/null +++ b/contrib/pg_walinspect/pg_walinspect.c @@ -0,0 +1,650 @@ +/*------------------------------------------------------------------------- + * + * pg_walinspect.c + * Functions to inspect contents of PostgreSQL Write-Ahead Log + * + * Copyright (c) 2022, PostgreSQL Global Development Group + * + * IDENTIFICATION + * contrib/pg_walinspect/pg_walinspect.c + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include "access/xlog.h" +#include "access/xlog_internal.h" +#include "access/xlogreader.h" +#include "access/xlogrecovery.h" +#include "access/xlogstats.h" +#include "access/xlogutils.h" +#include "funcapi.h" +#include "miscadmin.h" +#include "utils/builtins.h" +#include "utils/pg_lsn.h" + +/* + * NOTE: For any code change or issue fix here, it is highly recommended to + * give a thought about doing the same in pg_waldump tool as well. + */ + +PG_MODULE_MAGIC; + +PG_FUNCTION_INFO_V1(pg_get_wal_record_info); +PG_FUNCTION_INFO_V1(pg_get_wal_records_info); +PG_FUNCTION_INFO_V1(pg_get_wal_records_info_till_end_of_wal); +PG_FUNCTION_INFO_V1(pg_get_wal_stats); +PG_FUNCTION_INFO_V1(pg_get_wal_stats_till_end_of_wal); + +/* + * Struct holding information about the parameters that can be passed to + * GetWalStats. + */ +typedef struct GetWalStatsParams +{ + /* If true, generate statistics per-record instead of per-rmgr. */ + bool stats_per_record; +} GetWalStatsParams; + +typedef void (*GetWALDetailsCB) (FunctionCallInfo fcinfo, + XLogRecPtr start_lsn, + XLogRecPtr end_lsn, + void *params); + +static bool IsFutureLSN(XLogRecPtr lsn, XLogRecPtr *curr_lsn); +static XLogReaderState *InitXLogReaderState(XLogRecPtr lsn, + XLogRecPtr *first_record); +static XLogRecord *ReadNextXLogRecord(XLogReaderState *xlogreader, + XLogRecPtr first_record); +static void GetWALRecordInfo(XLogReaderState *record, XLogRecPtr lsn, + Datum *values, bool *nulls, uint32 ncols); +static void GetWALDetailsGuts(FunctionCallInfo fcinfo, bool till_end_of_wal, + GetWALDetailsCB wal_details_cb); +static void GetWALRecordsInfo(FunctionCallInfo fcinfo, XLogRecPtr start_lsn, + XLogRecPtr end_lsn, void *params); +static void GetXLogSummaryStats(XLogStats * stats, ReturnSetInfo *rsinfo, + Datum *values, bool *nulls, uint32 ncols, + bool stats_per_record); +static void FillXLogStatsRow(const char *name, uint64 n, uint64 total_count, + uint64 rec_len, uint64 total_rec_len, + uint64 fpi_len, uint64 total_fpi_len, + uint64 tot_len, uint64 total_len, + Datum *values, bool *nulls, uint32 ncols); +static void GetWalStats(FunctionCallInfo fcinfo, XLogRecPtr start_lsn, + XLogRecPtr end_lsn, void *params); + +/* + * Check if the given LSN is in future. Also, return the LSN up to which the + * server has WAL. + */ +static bool +IsFutureLSN(XLogRecPtr lsn, XLogRecPtr *curr_lsn) +{ + /* + * We determine the current LSN of the server similar to how page_read + * callback read_local_xlog_page_no_wait does. + */ + if (!RecoveryInProgress()) + *curr_lsn = GetFlushRecPtr(NULL); + else + *curr_lsn = GetXLogReplayRecPtr(NULL); + + Assert(!XLogRecPtrIsInvalid(*curr_lsn)); + + if (lsn >= *curr_lsn) + return true; + + return false; +} + +/* + * Intialize WAL reader and identify first valid LSN. + */ +static XLogReaderState * +InitXLogReaderState(XLogRecPtr lsn, XLogRecPtr *first_record) +{ + XLogReaderState *xlogreader; + + /* + * Reading WAL below the first page of the first sgements isn't allowed. + * This is a bootstrap WAL page and the page_read callback fails to read + * it. + */ + if (lsn < XLOG_BLCKSZ) + ereport(ERROR, + (errmsg("could not read WAL at LSN %X/%X", + LSN_FORMAT_ARGS(lsn)))); + + xlogreader = XLogReaderAllocate(wal_segment_size, NULL, + XL_ROUTINE(.page_read = &read_local_xlog_page_no_wait, + .segment_open = &wal_segment_open, + .segment_close = &wal_segment_close), + NULL); + + if (xlogreader == NULL) + ereport(ERROR, + (errcode(ERRCODE_OUT_OF_MEMORY), + errmsg("out of memory"), + errdetail("Failed while allocating a WAL reading processor."))); + + /* First find a valid recptr to start from. */ + *first_record = XLogFindNextRecord(xlogreader, lsn); + + if (XLogRecPtrIsInvalid(*first_record)) + ereport(ERROR, + (errmsg("could not find a valid record after %X/%X", + LSN_FORMAT_ARGS(lsn)))); + + return xlogreader; +} + +/* + * Read next WAL record. + */ +static XLogRecord * +ReadNextXLogRecord(XLogReaderState *xlogreader, XLogRecPtr first_record) +{ + XLogRecord *record; + char *errormsg; + + record = XLogReadRecord(xlogreader, &errormsg); + + if (record == NULL) + { + if (errormsg) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not read WAL at %X/%X: %s", + LSN_FORMAT_ARGS(first_record), errormsg))); + else + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not read WAL at %X/%X", + LSN_FORMAT_ARGS(first_record)))); + } + + return record; +} + +/* + * Get a single WAL record info. + */ +static void +GetWALRecordInfo(XLogReaderState *record, XLogRecPtr lsn, + Datum *values, bool *nulls, uint32 ncols) +{ + const char *id; + RmgrData desc; + uint32 fpi_len = 0; + StringInfoData rec_desc; + StringInfoData rec_blk_ref; + uint32 main_data_len; + int i = 0; + + desc = GetRmgr(XLogRecGetRmid(record)); + id = desc.rm_identify(XLogRecGetInfo(record)); + + if (id == NULL) + id = psprintf("UNKNOWN (%x)", XLogRecGetInfo(record) & ~XLR_INFO_MASK); + + initStringInfo(&rec_desc); + desc.rm_desc(&rec_desc, record); + + /* Block references. */ + initStringInfo(&rec_blk_ref); + XLogRecGetBlockRefInfo(record, NULL, &fpi_len, true, &rec_blk_ref); + + main_data_len = XLogRecGetDataLen(record); + + values[i++] = LSNGetDatum(lsn); + values[i++] = LSNGetDatum(record->EndRecPtr - 1); + values[i++] = LSNGetDatum(XLogRecGetPrev(record)); + values[i++] = TransactionIdGetDatum(XLogRecGetXid(record)); + values[i++] = CStringGetTextDatum(desc.rm_name); + values[i++] = CStringGetTextDatum(id); + values[i++] = UInt32GetDatum(XLogRecGetTotalLen(record)); + values[i++] = UInt32GetDatum(main_data_len); + values[i++] = UInt32GetDatum(fpi_len); + values[i++] = CStringGetTextDatum(rec_desc.data); + values[i++] = CStringGetTextDatum(rec_blk_ref.data); + + Assert(i == ncols); +} + +/* + * Get WAL record info. + * + * This function emits an error if a future WAL LSN i.e. WAL LSN the database + * system doesn't know about is specified. + */ +Datum +pg_get_wal_record_info(PG_FUNCTION_ARGS) +{ +#define PG_GET_WAL_RECORD_INFO_COLS 11 + Datum result; + Datum values[PG_GET_WAL_RECORD_INFO_COLS]; + bool nulls[PG_GET_WAL_RECORD_INFO_COLS]; + XLogRecPtr lsn; + XLogRecPtr curr_lsn; + XLogRecPtr first_record; + XLogReaderState *xlogreader; + TupleDesc tupdesc; + HeapTuple tuple; + + lsn = PG_GETARG_LSN(0); + + if (IsFutureLSN(lsn, &curr_lsn)) + { + /* + * GetFlushRecPtr or GetXLogReplayRecPtr gives "end+1" LSN of the last + * record flushed or replayed respectively. But let's use the LSN up + * to "end" in user facing message. + */ + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("cannot accept future input LSN"), + errdetail("Last WAL record on the database system ends at LSN %X/%X.", + LSN_FORMAT_ARGS(curr_lsn - 1)))); + } + + /* Build a tuple descriptor for our result type. */ + if (get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_COMPOSITE) + elog(ERROR, "return type must be a row type"); + + xlogreader = InitXLogReaderState(lsn, &first_record); + + Assert(xlogreader); + + (void) ReadNextXLogRecord(xlogreader, first_record); + + MemSet(values, 0, sizeof(values)); + MemSet(nulls, 0, sizeof(nulls)); + + GetWALRecordInfo(xlogreader, first_record, values, nulls, + PG_GET_WAL_RECORD_INFO_COLS); + + XLogReaderFree(xlogreader); + + tuple = heap_form_tuple(tupdesc, values, nulls); + result = HeapTupleGetDatum(tuple); + + PG_RETURN_DATUM(result); +#undef PG_GET_WAL_RECORD_INFO_COLS +} + +/* + * Get WAL details such as record info, stats using the passed in callback. + */ +static void +GetWALDetailsGuts(FunctionCallInfo fcinfo, bool till_end_of_wal, + GetWALDetailsCB wal_details_cb) +{ + XLogRecPtr start_lsn; + XLogRecPtr end_lsn; + XLogRecPtr curr_lsn; + GetWalStatsParams stats_params; + + start_lsn = PG_GETARG_LSN(0); + + /* If not till end of wal, end_lsn would have been specified. */ + if (!till_end_of_wal) + end_lsn = PG_GETARG_LSN(1); + + if (IsFutureLSN(start_lsn, &curr_lsn)) + { + /* + * GetFlushRecPtr or GetXLogReplayRecPtr gives "end+1" LSN of the last + * record flushed or replayed respectively. But let's use the LSN up + * to "end" in user facing message. + */ + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("cannot accept future start LSN"), + errdetail("Last WAL record on the database system ends at LSN %X/%X.", + LSN_FORMAT_ARGS(curr_lsn - 1)))); + } + + if (!till_end_of_wal && end_lsn >= curr_lsn) + { + /* + * GetFlushRecPtr or GetXLogReplayRecPtr gives "end+1" LSN of the last + * record flushed or replayed respectively. But let's use the LSN up + * to "end" in user facing message. + */ + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("cannot accept future end LSN"), + errdetail("Last WAL record on the database system ends at LSN %X/%X.", + LSN_FORMAT_ARGS(curr_lsn - 1)))); + } + else if (till_end_of_wal) + { + /* + * GetFlushRecPtr or GetXLogReplayRecPtr gives "end+1" LSN of the last + * record flushed or replayed respectively. But let's use the LSN up to + * "end". + */ + end_lsn = curr_lsn - 1; + } + + if (start_lsn >= end_lsn) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("WAL start LSN must be less than end LSN"))); + + if (wal_details_cb == GetWalStats) + { + MemSet(&stats_params, 0, sizeof(GetWalStatsParams)); + + if (!till_end_of_wal) + stats_params.stats_per_record = PG_GETARG_BOOL(2); + else + stats_params.stats_per_record = PG_GETARG_BOOL(1); + + wal_details_cb(fcinfo, start_lsn, end_lsn, (void *) &stats_params); + } + else if (wal_details_cb == GetWALRecordsInfo) + { + wal_details_cb(fcinfo, start_lsn, end_lsn, NULL); + } +} + +/* + * Get info and data of all WAL records between start LSN and end LSN. + */ +static void +GetWALRecordsInfo(FunctionCallInfo fcinfo, XLogRecPtr start_lsn, + XLogRecPtr end_lsn, void *params) +{ +#define PG_GET_WAL_RECORDS_INFO_COLS 11 + XLogRecPtr first_record; + XLogReaderState *xlogreader; + ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; + Datum values[PG_GET_WAL_RECORDS_INFO_COLS]; + bool nulls[PG_GET_WAL_RECORDS_INFO_COLS]; + + SetSingleFuncCall(fcinfo, 0); + + xlogreader = InitXLogReaderState(start_lsn, &first_record); + + Assert(xlogreader); + + MemSet(values, 0, sizeof(values)); + MemSet(nulls, 0, sizeof(nulls)); + + for (;;) + { + (void) ReadNextXLogRecord(xlogreader, first_record); + + /* + * Let's not show the record info if it is spanning more than the + * end_lsn. EndRecPtr is "end+1" of the last read record, hence + * use "end" here. + */ + if ((xlogreader->EndRecPtr - 1) <= end_lsn) + { + GetWALRecordInfo(xlogreader, xlogreader->currRecPtr, values, nulls, + PG_GET_WAL_RECORDS_INFO_COLS); + + tuplestore_putvalues(rsinfo->setResult, rsinfo->setDesc, + values, nulls); + } + + /* Exit loop if read up to end_lsn. */ + if (xlogreader->EndRecPtr >= end_lsn) + break; + + CHECK_FOR_INTERRUPTS(); + } + + XLogReaderFree(xlogreader); + +#undef PG_GET_WAL_RECORDS_INFO_COLS +} + +/* + * Get info and data of all WAL records between start LSN and end LSN. + * + * This function emits an error if a future start or end WAL LSN i.e. WAL LSN + * the database system doesn't know about is specified. + */ +Datum +pg_get_wal_records_info(PG_FUNCTION_ARGS) +{ + GetWALDetailsGuts(fcinfo, false, GetWALRecordsInfo); + + PG_RETURN_VOID(); +} + +/* + * Get info and data of all WAL records from start LSN till end of WAL. + * + * This function emits an error if a future start i.e. WAL LSN the database + * system doesn't know about is specified. + */ +Datum +pg_get_wal_records_info_till_end_of_wal(PG_FUNCTION_ARGS) +{ + GetWALDetailsGuts(fcinfo, true, GetWALRecordsInfo); + + PG_RETURN_VOID(); +} + +/* + * Fill single row of record counts and sizes for an rmgr or record. + */ +static void +FillXLogStatsRow(const char *name, + uint64 n, uint64 total_count, + uint64 rec_len, uint64 total_rec_len, + uint64 fpi_len, uint64 total_fpi_len, + uint64 tot_len, uint64 total_len, + Datum *values, bool *nulls, uint32 ncols) +{ + double n_pct, + rec_len_pct, + fpi_len_pct, + tot_len_pct; + int i = 0; + + n_pct = 0; + if (total_count != 0) + n_pct = 100 * (double) n / total_count; + + rec_len_pct = 0; + if (total_rec_len != 0) + rec_len_pct = 100 * (double) rec_len / total_rec_len; + + fpi_len_pct = 0; + if (total_fpi_len != 0) + fpi_len_pct = 100 * (double) fpi_len / total_fpi_len; + + tot_len_pct = 0; + if (total_len != 0) + tot_len_pct = 100 * (double) tot_len / total_len; + + values[i++] = CStringGetTextDatum(name); + values[i++] = Int64GetDatum(n); + values[i++] = Float4GetDatum(n_pct); + values[i++] = Int64GetDatum(rec_len); + values[i++] = Float4GetDatum(rec_len_pct); + values[i++] = Int64GetDatum(fpi_len); + values[i++] = Float4GetDatum(fpi_len_pct); + values[i++] = Int64GetDatum(tot_len); + values[i++] = Float4GetDatum(tot_len_pct); + + Assert(i == ncols); +} + +/* + * Get summary statistics about the records seen so far. + */ +static void +GetXLogSummaryStats(XLogStats *stats, ReturnSetInfo *rsinfo, + Datum *values, bool *nulls, uint32 ncols, + bool stats_per_record) +{ + uint64 total_count = 0; + uint64 total_rec_len = 0; + uint64 total_fpi_len = 0; + uint64 total_len = 0; + int ri; + + /* + * Each row shows its percentages of the total, so make a first pass to + * calculate column totals. + */ + for (ri = 0; ri <= RM_MAX_ID; ri++) + { + if (!RmgrIdIsValid(ri)) + continue; + + total_count += stats->rmgr_stats[ri].count; + total_rec_len += stats->rmgr_stats[ri].rec_len; + total_fpi_len += stats->rmgr_stats[ri].fpi_len; + } + total_len = total_rec_len + total_fpi_len; + + for (ri = 0; ri <= RM_MAX_ID; ri++) + { + uint64 count; + uint64 rec_len; + uint64 fpi_len; + uint64 tot_len; + RmgrData desc; + + if (!RmgrIdIsValid(ri)) + continue; + + if (!RmgrIdExists(ri)) + continue; + + desc = GetRmgr(ri); + + if (stats_per_record) + { + int rj; + + for (rj = 0; rj < MAX_XLINFO_TYPES; rj++) + { + const char *id; + + count = stats->record_stats[ri][rj].count; + rec_len = stats->record_stats[ri][rj].rec_len; + fpi_len = stats->record_stats[ri][rj].fpi_len; + tot_len = rec_len + fpi_len; + + /* Skip undefined combinations and ones that didn't occur */ + if (count == 0) + continue; + + /* the upper four bits in xl_info are the rmgr's */ + id = desc.rm_identify(rj << 4); + if (id == NULL) + id = psprintf("UNKNOWN (%x)", rj << 4); + + FillXLogStatsRow(psprintf("%s/%s", desc.rm_name, id), count, + total_count, rec_len, total_rec_len, fpi_len, + total_fpi_len, tot_len, total_len, + values, nulls, ncols); + + tuplestore_putvalues(rsinfo->setResult, rsinfo->setDesc, + values, nulls); + } + } + else + { + count = stats->rmgr_stats[ri].count; + rec_len = stats->rmgr_stats[ri].rec_len; + fpi_len = stats->rmgr_stats[ri].fpi_len; + tot_len = rec_len + fpi_len; + + FillXLogStatsRow(desc.rm_name, count, total_count, rec_len, + total_rec_len, fpi_len, total_fpi_len, tot_len, + total_len, values, nulls, ncols); + + tuplestore_putvalues(rsinfo->setResult, rsinfo->setDesc, + values, nulls); + } + } +} + +/* + * Get WAL stats between start LSN and end LSN. + */ +static void +GetWalStats(FunctionCallInfo fcinfo, XLogRecPtr start_lsn, + XLogRecPtr end_lsn, void *params) +{ +#define PG_GET_WAL_STATS_COLS 9 + XLogRecPtr first_record; + XLogReaderState *xlogreader; + XLogStats stats; + ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; + Datum values[PG_GET_WAL_STATS_COLS]; + bool nulls[PG_GET_WAL_STATS_COLS]; + + SetSingleFuncCall(fcinfo, 0); + + xlogreader = InitXLogReaderState(start_lsn, &first_record); + + MemSet(&stats, 0, sizeof(stats)); + + for (;;) + { + (void) ReadNextXLogRecord(xlogreader, first_record); + + /* + * Let's not show the record info if it is spanning more than the + * end_lsn. EndRecPtr is "end+1" of the last read record, hence + * use "end" here. + */ + if ((xlogreader->EndRecPtr - 1) <= end_lsn) + XLogRecStoreStats(&stats, xlogreader); + + /* Exit loop if read up to end_lsn. */ + if (xlogreader->EndRecPtr >= end_lsn) + break; + + CHECK_FOR_INTERRUPTS(); + } + + XLogReaderFree(xlogreader); + + MemSet(values, 0, sizeof(values)); + MemSet(nulls, 0, sizeof(nulls)); + + GetXLogSummaryStats(&stats, rsinfo, values, nulls, + PG_GET_WAL_STATS_COLS, + ((GetWalStatsParams *)params)->stats_per_record); + +#undef PG_GET_WAL_STATS_COLS +} + +/* + * Get stats of all WAL records between start LSN and end LSN. + * + * This function emits an error if a future start or end WAL LSN i.e. WAL LSN + * the database system doesn't know about is specified. + */ +Datum +pg_get_wal_stats(PG_FUNCTION_ARGS) +{ + GetWALDetailsGuts(fcinfo, false, GetWalStats); + + PG_RETURN_VOID(); +} + +/* + * Get stats of all WAL records from start LSN till end of WAL. + * + * This function emits an error if a future start i.e. WAL LSN the database + * system doesn't know about is specified. + */ +Datum +pg_get_wal_stats_till_end_of_wal(PG_FUNCTION_ARGS) +{ + GetWALDetailsGuts(fcinfo, true, GetWalStats); + + PG_RETURN_VOID(); +} diff --git a/contrib/pg_walinspect/pg_walinspect.control b/contrib/pg_walinspect/pg_walinspect.control new file mode 100644 index 0000000000..017e56a2bb --- /dev/null +++ b/contrib/pg_walinspect/pg_walinspect.control @@ -0,0 +1,5 @@ +# pg_walinspect extension +comment = 'functions to inspect contents of PostgreSQL Write-Ahead Log' +default_version = '1.0' +module_pathname = '$libdir/pg_walinspect' +relocatable = true diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index 5862d9dc75..dea1f877ae 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -1320,13 +1320,6 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr, return true; } -#ifdef FRONTEND -/* - * Functions that are currently not needed in the backend, but are better - * implemented inside xlogreader.c because of the internal facilities available - * here. - */ - /* * Find the first record with an lsn >= RecPtr. * @@ -1447,6 +1440,12 @@ err: return InvalidXLogRecPtr; } +#ifdef FRONTEND +/* + * Functions that are currently not needed in the backend, but are better + * implemented inside xlogreader.c because of the internal facilities available + * here. + */ #endif /* FRONTEND */ /* diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c index bb2d3ec991..b5d34c61e6 100644 --- a/src/backend/access/transam/xlogutils.c +++ b/src/backend/access/transam/xlogutils.c @@ -80,6 +80,10 @@ typedef struct xl_invalid_page static HTAB *invalid_page_tab = NULL; +static int +read_local_xlog_page_guts(XLogReaderState *state, XLogRecPtr targetPagePtr, + int reqLen, XLogRecPtr targetRecPtr, + char *cur_page, bool wait_for_wal); /* Report a reference to an invalid page */ static void @@ -870,6 +874,31 @@ wal_segment_close(XLogReaderState *state) int read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen, XLogRecPtr targetRecPtr, char *cur_page) +{ + return read_local_xlog_page_guts(state, targetPagePtr, reqLen, + targetRecPtr, cur_page, true); +} + +/* + * Same as read_local_xlog_page except that it doesn't wait for future WAL + * to be available. + */ +int +read_local_xlog_page_no_wait(XLogReaderState *state, XLogRecPtr targetPagePtr, + int reqLen, XLogRecPtr targetRecPtr, + char *cur_page) +{ + return read_local_xlog_page_guts(state, targetPagePtr, reqLen, + targetRecPtr, cur_page, false); +} + +/* + * Implementation of read_local_xlog_page and its no wait version. + */ +static int +read_local_xlog_page_guts(XLogReaderState *state, XLogRecPtr targetPagePtr, + int reqLen, XLogRecPtr targetRecPtr, + char *cur_page, bool wait_for_wal) { XLogRecPtr read_upto, loc; @@ -925,6 +954,10 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, if (loc <= read_upto) break; + /* If asked, let's not wait for future WAL. */ + if (!wait_for_wal) + break; + CHECK_FOR_INTERRUPTS(); pg_usleep(1000L); } diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c index d918845cb2..54d765b331 100644 --- a/src/bin/pg_waldump/pg_waldump.c +++ b/src/bin/pg_waldump/pg_waldump.c @@ -27,6 +27,11 @@ #include "getopt_long.h" #include "rmgrdesc.h" +/* + * NOTE: For any code change or issue fix here, it is highly recommended to + * give a thought about doing the same in pg_walinspect contrib module as well. + */ + static const char *progname; static int WalSegSz; diff --git a/src/include/access/xlog.h b/src/include/access/xlog.h index e302bd102c..5e1e3446ae 100644 --- a/src/include/access/xlog.h +++ b/src/include/access/xlog.h @@ -31,7 +31,7 @@ extern XLogRecPtr XactLastRecEnd; extern PGDLLIMPORT XLogRecPtr XactLastCommitEnd; /* these variables are GUC parameters related to XLOG */ -extern int wal_segment_size; +extern PGDLLIMPORT int wal_segment_size; extern int min_wal_size_mb; extern int max_wal_size_mb; extern int wal_keep_size_mb; diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index d0206f7c74..fd8d51af7d 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -320,7 +320,7 @@ typedef struct RmgrData struct XLogRecordBuffer *buf); } RmgrData; -extern RmgrData RmgrTable[]; +extern PGDLLIMPORT RmgrData RmgrTable[]; extern void RmgrStartup(void); extern void RmgrCleanup(void); extern void RmgrNotFound(RmgrId rmid); diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h index d8eb857611..727e9fe971 100644 --- a/src/include/access/xlogreader.h +++ b/src/include/access/xlogreader.h @@ -344,9 +344,7 @@ extern void XLogReaderSetDecodeBuffer(XLogReaderState *state, /* Position the XLogReader to given record */ extern void XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr); -#ifdef FRONTEND extern XLogRecPtr XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr); -#endif /* FRONTEND */ /* Return values from XLogPageReadCB. */ typedef enum XLogPageReadResult diff --git a/src/include/access/xlogutils.h b/src/include/access/xlogutils.h index ff40f96e42..3746e31e40 100644 --- a/src/include/access/xlogutils.h +++ b/src/include/access/xlogutils.h @@ -93,6 +93,10 @@ extern void FreeFakeRelcacheEntry(Relation fakerel); extern int read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen, XLogRecPtr targetRecPtr, char *cur_page); +extern int read_local_xlog_page_no_wait(XLogReaderState *state, + XLogRecPtr targetPagePtr, int reqLen, + XLogRecPtr targetRecPtr, + char *cur_page); extern void wal_segment_open(XLogReaderState *state, XLogSegNo nextSegNo, TimeLineID *tli_p); -- 2.25.1 [application/x-patch] v18-0003-pg_walinspect-tests.patch (9.7K, ../../CALj2ACV7cXcCO-Ecyc9+_f71WeN-_HNqk=HJ7Ymvp3dXqN_s1A@mail.gmail.com/4-v18-0003-pg_walinspect-tests.patch) download | inline diff: From 67c34743edf3a8dbbec8b7eafa8f7b9be91c8dd1 Mon Sep 17 00:00:00 2001 From: Bharath Rupireddy <[email protected]> Date: Thu, 7 Apr 2022 06:58:02 +0000 Subject: [PATCH v18] pg_walinspect tests --- .../pg_walinspect/expected/pg_walinspect.out | 145 ++++++++++++++++++ contrib/pg_walinspect/sql/pg_walinspect.sql | 107 +++++++++++++ 2 files changed, 252 insertions(+) create mode 100644 contrib/pg_walinspect/expected/pg_walinspect.out create mode 100644 contrib/pg_walinspect/sql/pg_walinspect.sql diff --git a/contrib/pg_walinspect/expected/pg_walinspect.out b/contrib/pg_walinspect/expected/pg_walinspect.out new file mode 100644 index 0000000000..53d3146fe1 --- /dev/null +++ b/contrib/pg_walinspect/expected/pg_walinspect.out @@ -0,0 +1,145 @@ +CREATE EXTENSION pg_walinspect; +CREATE TABLE sample_tbl(col1 int, col2 int); +SELECT pg_current_wal_lsn() AS wal_lsn1 \gset +INSERT INTO sample_tbl SELECT * FROM generate_series(1, 2); +SELECT pg_current_wal_lsn() AS wal_lsn2 \gset +INSERT INTO sample_tbl SELECT * FROM generate_series(1, 2); +-- =================================================================== +-- Tests for input validation +-- =================================================================== +SELECT COUNT(*) >= 0 AS ok FROM pg_get_wal_records_info(:'wal_lsn2', :'wal_lsn1'); -- ERROR +ERROR: WAL start LSN must be less than end LSN +SELECT COUNT(*) >= 0 AS ok FROM pg_get_wal_stats(:'wal_lsn2', :'wal_lsn1'); -- ERROR +ERROR: WAL start LSN must be less than end LSN +-- =================================================================== +-- Tests for all function executions +-- =================================================================== +SELECT COUNT(*) >= 0 AS ok FROM pg_get_wal_record_info(:'wal_lsn1'); + ok +---- + t +(1 row) + +SELECT COUNT(*) >= 0 AS ok FROM pg_get_wal_records_info(:'wal_lsn1', :'wal_lsn2'); + ok +---- + t +(1 row) + +SELECT COUNT(*) >= 0 AS ok FROM pg_get_wal_records_info_till_end_of_wal(:'wal_lsn1'); + ok +---- + t +(1 row) + +SELECT COUNT(*) >= 0 AS ok FROM pg_get_wal_stats(:'wal_lsn1', :'wal_lsn2'); + ok +---- + t +(1 row) + +SELECT COUNT(*) >= 0 AS ok FROM pg_get_wal_stats_till_end_of_wal(:'wal_lsn1'); + ok +---- + t +(1 row) + +-- =================================================================== +-- Tests for filtering out WAL records of a particular table +-- =================================================================== +SELECT oid AS sample_tbl_oid FROM pg_class WHERE relname = 'sample_tbl' \gset +SELECT COUNT(*) >= 1 AS ok FROM pg_get_wal_records_info(:'wal_lsn1', :'wal_lsn2') + WHERE block_ref LIKE concat('%', :'sample_tbl_oid', '%') AND resource_manager = 'Heap'; + ok +---- + t +(1 row) + +-- =================================================================== +-- Tests for permissions +-- =================================================================== +CREATE ROLE regress_pg_walinspect; +SELECT has_function_privilege('regress_pg_walinspect', + 'pg_get_wal_record_info(pg_lsn)', 'EXECUTE'); -- no + has_function_privilege +------------------------ + f +(1 row) + +SELECT has_function_privilege('regress_pg_walinspect', + 'pg_get_wal_records_info(pg_lsn, pg_lsn) ', 'EXECUTE'); -- no + has_function_privilege +------------------------ + f +(1 row) + +SELECT has_function_privilege('regress_pg_walinspect', + 'pg_get_wal_stats(pg_lsn, pg_lsn, boolean) ', 'EXECUTE'); -- no + has_function_privilege +------------------------ + f +(1 row) + +-- Functions accessible by users with role pg_read_server_files +GRANT pg_read_server_files TO regress_pg_walinspect; +SELECT has_function_privilege('regress_pg_walinspect', + 'pg_get_wal_record_info(pg_lsn)', 'EXECUTE'); -- yes + has_function_privilege +------------------------ + t +(1 row) + +SELECT has_function_privilege('regress_pg_walinspect', + 'pg_get_wal_records_info(pg_lsn, pg_lsn) ', 'EXECUTE'); -- yes + has_function_privilege +------------------------ + t +(1 row) + +SELECT has_function_privilege('regress_pg_walinspect', + 'pg_get_wal_stats(pg_lsn, pg_lsn, boolean) ', 'EXECUTE'); -- yes + has_function_privilege +------------------------ + t +(1 row) + +REVOKE pg_read_server_files FROM regress_pg_walinspect; +-- Superuser can grant execute to other users +GRANT EXECUTE ON FUNCTION pg_get_wal_record_info(pg_lsn) + TO regress_pg_walinspect; +GRANT EXECUTE ON FUNCTION pg_get_wal_records_info(pg_lsn, pg_lsn) + TO regress_pg_walinspect; +GRANT EXECUTE ON FUNCTION pg_get_wal_stats(pg_lsn, pg_lsn, boolean) + TO regress_pg_walinspect; +SELECT has_function_privilege('regress_pg_walinspect', + 'pg_get_wal_record_info(pg_lsn)', 'EXECUTE'); -- yes + has_function_privilege +------------------------ + t +(1 row) + +SELECT has_function_privilege('regress_pg_walinspect', + 'pg_get_wal_records_info(pg_lsn, pg_lsn) ', 'EXECUTE'); -- yes + has_function_privilege +------------------------ + t +(1 row) + +SELECT has_function_privilege('regress_pg_walinspect', + 'pg_get_wal_stats(pg_lsn, pg_lsn, boolean) ', 'EXECUTE'); -- yes + has_function_privilege +------------------------ + t +(1 row) + +REVOKE EXECUTE ON FUNCTION pg_get_wal_record_info(pg_lsn) + FROM regress_pg_walinspect; +REVOKE EXECUTE ON FUNCTION pg_get_wal_records_info(pg_lsn, pg_lsn) + FROM regress_pg_walinspect; +REVOKE EXECUTE ON FUNCTION pg_get_wal_stats(pg_lsn, pg_lsn, boolean) + FROM regress_pg_walinspect; +-- =================================================================== +-- Clean up +-- =================================================================== +DROP ROLE regress_pg_walinspect; +DROP TABLE sample_tbl; diff --git a/contrib/pg_walinspect/sql/pg_walinspect.sql b/contrib/pg_walinspect/sql/pg_walinspect.sql new file mode 100644 index 0000000000..6e120c472b --- /dev/null +++ b/contrib/pg_walinspect/sql/pg_walinspect.sql @@ -0,0 +1,107 @@ +CREATE EXTENSION pg_walinspect; + +CREATE TABLE sample_tbl(col1 int, col2 int); + +SELECT pg_current_wal_lsn() AS wal_lsn1 \gset + +INSERT INTO sample_tbl SELECT * FROM generate_series(1, 2); + +SELECT pg_current_wal_lsn() AS wal_lsn2 \gset + +INSERT INTO sample_tbl SELECT * FROM generate_series(1, 2); + +-- =================================================================== +-- Tests for input validation +-- =================================================================== + +SELECT COUNT(*) >= 0 AS ok FROM pg_get_wal_records_info(:'wal_lsn2', :'wal_lsn1'); -- ERROR + +SELECT COUNT(*) >= 0 AS ok FROM pg_get_wal_stats(:'wal_lsn2', :'wal_lsn1'); -- ERROR + +-- =================================================================== +-- Tests for all function executions +-- =================================================================== + +SELECT COUNT(*) >= 0 AS ok FROM pg_get_wal_record_info(:'wal_lsn1'); + +SELECT COUNT(*) >= 0 AS ok FROM pg_get_wal_records_info(:'wal_lsn1', :'wal_lsn2'); + +SELECT COUNT(*) >= 0 AS ok FROM pg_get_wal_records_info_till_end_of_wal(:'wal_lsn1'); + +SELECT COUNT(*) >= 0 AS ok FROM pg_get_wal_stats(:'wal_lsn1', :'wal_lsn2'); + +SELECT COUNT(*) >= 0 AS ok FROM pg_get_wal_stats_till_end_of_wal(:'wal_lsn1'); + +-- =================================================================== +-- Tests for filtering out WAL records of a particular table +-- =================================================================== + +SELECT oid AS sample_tbl_oid FROM pg_class WHERE relname = 'sample_tbl' \gset + +SELECT COUNT(*) >= 1 AS ok FROM pg_get_wal_records_info(:'wal_lsn1', :'wal_lsn2') + WHERE block_ref LIKE concat('%', :'sample_tbl_oid', '%') AND resource_manager = 'Heap'; + +-- =================================================================== +-- Tests for permissions +-- =================================================================== +CREATE ROLE regress_pg_walinspect; + +SELECT has_function_privilege('regress_pg_walinspect', + 'pg_get_wal_record_info(pg_lsn)', 'EXECUTE'); -- no + +SELECT has_function_privilege('regress_pg_walinspect', + 'pg_get_wal_records_info(pg_lsn, pg_lsn) ', 'EXECUTE'); -- no + +SELECT has_function_privilege('regress_pg_walinspect', + 'pg_get_wal_stats(pg_lsn, pg_lsn, boolean) ', 'EXECUTE'); -- no + +-- Functions accessible by users with role pg_read_server_files + +GRANT pg_read_server_files TO regress_pg_walinspect; + +SELECT has_function_privilege('regress_pg_walinspect', + 'pg_get_wal_record_info(pg_lsn)', 'EXECUTE'); -- yes + +SELECT has_function_privilege('regress_pg_walinspect', + 'pg_get_wal_records_info(pg_lsn, pg_lsn) ', 'EXECUTE'); -- yes + +SELECT has_function_privilege('regress_pg_walinspect', + 'pg_get_wal_stats(pg_lsn, pg_lsn, boolean) ', 'EXECUTE'); -- yes + +REVOKE pg_read_server_files FROM regress_pg_walinspect; + +-- Superuser can grant execute to other users +GRANT EXECUTE ON FUNCTION pg_get_wal_record_info(pg_lsn) + TO regress_pg_walinspect; + +GRANT EXECUTE ON FUNCTION pg_get_wal_records_info(pg_lsn, pg_lsn) + TO regress_pg_walinspect; + +GRANT EXECUTE ON FUNCTION pg_get_wal_stats(pg_lsn, pg_lsn, boolean) + TO regress_pg_walinspect; + +SELECT has_function_privilege('regress_pg_walinspect', + 'pg_get_wal_record_info(pg_lsn)', 'EXECUTE'); -- yes + +SELECT has_function_privilege('regress_pg_walinspect', + 'pg_get_wal_records_info(pg_lsn, pg_lsn) ', 'EXECUTE'); -- yes + +SELECT has_function_privilege('regress_pg_walinspect', + 'pg_get_wal_stats(pg_lsn, pg_lsn, boolean) ', 'EXECUTE'); -- yes + +REVOKE EXECUTE ON FUNCTION pg_get_wal_record_info(pg_lsn) + FROM regress_pg_walinspect; + +REVOKE EXECUTE ON FUNCTION pg_get_wal_records_info(pg_lsn, pg_lsn) + FROM regress_pg_walinspect; + +REVOKE EXECUTE ON FUNCTION pg_get_wal_stats(pg_lsn, pg_lsn, boolean) + FROM regress_pg_walinspect; + +-- =================================================================== +-- Clean up +-- =================================================================== + +DROP ROLE regress_pg_walinspect; + +DROP TABLE sample_tbl; -- 2.25.1 [application/x-patch] v18-0004-pg_walinspect-docs.patch (15.1K, ../../CALj2ACV7cXcCO-Ecyc9+_f71WeN-_HNqk=HJ7Ymvp3dXqN_s1A@mail.gmail.com/5-v18-0004-pg_walinspect-docs.patch) download | inline diff: From e146d961755294f8064d6afd0c24a6c6d77e11b4 Mon Sep 17 00:00:00 2001 From: Bharath Rupireddy <[email protected]> Date: Thu, 7 Apr 2022 06:58:45 +0000 Subject: [PATCH v18] pg_walinspect docs --- doc/src/sgml/contrib.sgml | 1 + doc/src/sgml/filelist.sgml | 1 + doc/src/sgml/pgwalinspect.sgml | 244 +++++++++++++++++++++++++++++++++ 3 files changed, 246 insertions(+) create mode 100644 doc/src/sgml/pgwalinspect.sgml diff --git a/doc/src/sgml/contrib.sgml b/doc/src/sgml/contrib.sgml index 1e42ce1a7f..4e7b87a42f 100644 --- a/doc/src/sgml/contrib.sgml +++ b/doc/src/sgml/contrib.sgml @@ -131,6 +131,7 @@ CREATE EXTENSION <replaceable>module_name</replaceable>; &pgsurgery; &pgtrgm; &pgvisibility; + &pgwalinspect; &postgres-fdw; &seg; &sepgsql; diff --git a/doc/src/sgml/filelist.sgml b/doc/src/sgml/filelist.sgml index 7dea670969..1e82cb2d3d 100644 --- a/doc/src/sgml/filelist.sgml +++ b/doc/src/sgml/filelist.sgml @@ -148,6 +148,7 @@ <!ENTITY pgsurgery SYSTEM "pgsurgery.sgml"> <!ENTITY pgtrgm SYSTEM "pgtrgm.sgml"> <!ENTITY pgvisibility SYSTEM "pgvisibility.sgml"> +<!ENTITY pgwalinspect SYSTEM "pgwalinspect.sgml"> <!ENTITY postgres-fdw SYSTEM "postgres-fdw.sgml"> <!ENTITY seg SYSTEM "seg.sgml"> <!ENTITY contrib-spi SYSTEM "contrib-spi.sgml"> diff --git a/doc/src/sgml/pgwalinspect.sgml b/doc/src/sgml/pgwalinspect.sgml new file mode 100644 index 0000000000..e44874b87b --- /dev/null +++ b/doc/src/sgml/pgwalinspect.sgml @@ -0,0 +1,244 @@ +<!-- doc/src/sgml/pgwalinspect.sgml --> + +<sect1 id="pgwalinspect" xreflabel="pg_walinspect"> + <title>pg_walinspect</title> + + <indexterm zone="pgwalinspect"> + <primary>pg_walinspect</primary> + </indexterm> + + <para> + The <filename>pg_walinspect</filename> module provides functions that allow + you to inspect the contents of write-ahead log of <productname>PostgreSQL</productname> + database cluster at a low level, which is useful for debugging or analytical + or reporting or educational purposes. + </para> + + <para> + All the functions of this module will provide the WAL information using the + current server's timeline ID. + </para> + + <para> + By default, use of these functions is restricted to superusers and members of + the <literal>pg_read_server_files</literal> role. Access may be granted by + superusers to others using <command>GRANT</command>. + </para> + + <sect2> + <title>General Functions</title> + + <variablelist> + <varlistentry> + <term> + <function> + pg_get_wal_record_info(in_lsn pg_lsn, + start_lsn OUT pg_lsn, + end_lsn OUT pg_lsn, + prev_lsn OUT pg_lsn, + xid OUT xid, + resource_manager OUT text, + record_type OUT text, + record_length OUT int4, + main_data_length OUT int4, + fpi_length OUT int4, + description OUT text, + block_ref OUT text) + </function> + </term> + + <listitem> + <para> + Gets WAL record information of a given LSN. If the given LSN isn't + containing a valid WAL record, it gives the information of the next + available valid WAL record. This function emits an error if a future (the + LSN database system doesn't know about) <replaceable>in_lsn</replaceable> + is specified. + </para> + </listitem> + </varlistentry> + + <varlistentry> + <term> + <function> + pg_get_wal_records_info(start_lsn pg_lsn, + end_lsn pg_lsn, + start_lsn OUT pg_lsn, + end_lsn OUT pg_lsn, + prev_lsn OUT pg_lsn, + xid OUT xid, + resource_manager OUT text, + record_type OUT text, + record_length OUT int4, + main_data_length OUT int4, + fpi_length OUT int4, + description OUT text, + block_ref OUT text) + returns setof record + </function> + </term> + + <listitem> + <para> + Gets information of all the valid WAL records between + <replaceable>start_lsn</replaceable> and <replaceable>end_lsn</replaceable>. + Returns one row per each valid WAL record. This function emits an error + if a future (the LSN database system doesn't know about) + <replaceable>start_lsn</replaceable> or <replaceable>end_lsn</replaceable> + is specified. For example, usage of the function is as follows: +<screen> +postgres=# select start_lsn, end_lsn, prev_lsn, xid, resource_manager, record_type, record_length, main_data_length, fpi_length, description from pg_get_wal_records_info('0/14FBA30', '0/15011D7'); + start_lsn | end_lsn | prev_lsn | xid | resource_manager | record_type | record_length | main_data_length | fpi_length | description +-----------+-----------+-----------+-----+------------------+---------------+---------------+------------------+------------+--------------------------------------------------------- + 0/14FBA30 | 0/14FBA67 | 0/14FB9F8 | 0 | Heap2 | PRUNE | 56 | 8 | 0 | latestRemovedXid 0 nredirected 0 ndead 1 + 0/14FBA68 | 0/14FBA9F | 0/14FBA30 | 0 | Standby | RUNNING_XACTS | 50 | 24 | 0 | nextXid 723 latestCompletedXid 722 oldestRunningXid 723 + 0/14FBAA0 | 0/14FBACF | 0/14FBA68 | 0 | Storage | CREATE | 42 | 16 | 0 | base/5/16390 + 0/14FBAD0 | 0/14FC117 | 0/14FBAA0 | 723 | Heap | INSERT | 1582 | 3 | 1528 | off 8 flags 0x01 + 0/14FC118 | 0/14FD487 | 0/14FBAD0 | 723 | Btree | INSERT_LEAF | 4973 | 2 | 4920 | off 244 + 0/14FD488 | 0/14FEFCF | 0/14FC118 | 723 | Btree | INSERT_LEAF | 6953 | 2 | 6900 | off 126 + 0/14FEFD0 | 0/14FF027 | 0/14FD488 | 723 | Heap2 | MULTI_INSERT | 85 | 6 | 0 | 1 tuples flags 0x02 + 0/14FF028 | 0/14FF06F | 0/14FEFD0 | 723 | Btree | INSERT_LEAF | 72 | 2 | 0 | off 155 + 0/14FF070 | 0/14FF0B7 | 0/14FF028 | 723 | Btree | INSERT_LEAF | 72 | 2 | 0 | off 134 + 0/14FF0B8 | 0/14FF18F | 0/14FF070 | 723 | Heap | INSERT | 211 | 3 | 0 | off 9 flags 0x00 + 0/14FF190 | 0/14FF1CF | 0/14FF0B8 | 723 | Btree | INSERT_LEAF | 64 | 2 | 0 | off 244 + 0/14FF1D0 | 0/15010B7 | 0/14FF190 | 723 | Btree | SPLIT_L | 7885 | 10 | 4136 | level 0, firstrightoff 120, newitemoff 47, postingoff 0 + 0/15010B8 | 0/150117F | 0/14FF1D0 | 723 | Btree | INSERT_UPPER | 197 | 2 | 136 | off 2 + 0/1501180 | 0/15011D7 | 0/15010B8 | 723 | Heap2 | MULTI_INSERT | 85 | 6 | 0 | 1 tuples flags 0x02 +(14 rows) +</screen> + </para> + </listitem> + </varlistentry> + + <varlistentry> + <term> + <function> + pg_get_wal_records_info_till_end_of_wal(start_lsn pg_lsn, + start_lsn OUT pg_lsn, + end_lsn OUT pg_lsn, + prev_lsn OUT pg_lsn, + xid OUT xid, + resource_manager OUT text, + record_type OUT text, + record_length OUT int4, + main_data_length OUT int4, + fpi_length OUT int4, + description OUT text, + block_ref OUT text) + returns setof record + </function> + </term> + + <listitem> + <para> + This function is same as <function>pg_get_wal_records_info()</function> + except that it gets information of all the valid WAL records from + <replaceable>start_lsn</replaceable> till end of WAL. + </para> + </listitem> + </varlistentry> + + <varlistentry> + <term> + <function> + pg_get_wal_stats(start_lsn pg_lsn, + end_lsn pg_lsn, + per_record boolean DEFAULT false, + "resource_manager/record_type" OUT text, + count OUT int8, + count_percentage OUT float4, + record_length OUT int8, + record_length_percentage OUT float4, + fpi_length OUT int8, + fpi_length_percentage OUT float4, + combined_size OUT int8, + combined_size_percentage OUT float4) + returns setof record + </function> + </term> + + <listitem> + <para> + Gets statistics of all the valid WAL records between + <replaceable>start_lsn</replaceable> and + <replaceable>end_lsn</replaceable>. By default, it returns one row per + <replaceable>resource_manager</replaceable> type. When + <replaceable>per_record</replaceable> is set to <literal>true</literal>, + it returns one row per <replaceable>record_type</replaceable>. This + function emits an error if a future (the LSN database system doesn't know + about) <replaceable>start_lsn</replaceable> or <replaceable>end_lsn</replaceable> + is specified. For example, usage of the function is as follows: +<screen> +postgres=# select * from pg_get_wal_stats('0/12FBA30', '0/15011D7') where count > 0; + resource_manager/record_type | count | count_percentage | record_size | record_size_percentage | fpi_size | fpi_size_percentage | combined_size | combined_size_percentage +------------------------------+-------+------------------+-------------+------------------------+----------+---------------------+---------------+-------------------------- + XLOG | 12 | 0.13002492 | 1024 | 2.3833392e-05 | 352 | 0.06136488 | 1376 | 3.2021846e-05 + Transaction | 188 | 2.0370572 | 62903 | 0.0014640546 | 0 | 0 | 62903 | 0.0014638591 + Storage | 13 | 0.14086033 | 546 | 1.2708038e-05 | 0 | 0 | 546 | 1.2706342e-05 + Database | 2 | 0.02167082 | 84 | 1.955083e-06 | 0 | 0 | 84 | 1.954822e-06 + Standby | 219 | 2.3729548 | 15830 | 0.00036844003 | 0 | 0 | 15830 | 0.00036839084 + Heap2 | 1905 | 20.641457 | 384619 | 0.0089519285 | 364472 | 63.53915 | 749091 | 0.017432613 + Heap | 1319 | 14.291906 | 621997 | 0.014476853 | 145232 | 25.318592 | 767229 | 0.017854715 + Btree | 5571 | 60.36407 | 4295405999 | 99.9747 | 63562 | 11.0808935 | 4295469561 | 99.96284 +(8 rows) +</screen> + +With <replaceable>per_record</replaceable> passed as <literal>true</literal>: + +<screen> +postgres=# select * from pg_get_wal_stats('0/14FBA30', '0/15011D7', true) where count > 0; + resource_manager/record_type | count | count_percentage | record_size | record_size_percentage | fpi_size | fpi_size_percentage | combined_size | combined_size_percentage +------------------------------+-------+------------------+-------------+------------------------+----------+---------------------+---------------+-------------------------- + Storage/CREATE | 1 | 7.142857 | 42 | 0.8922881 | 0 | 0 | 42 | 0.18811305 + Standby/RUNNING_XACTS | 1 | 7.142857 | 50 | 1.0622478 | 0 | 0 | 50 | 0.2239441 + Heap2/PRUNE | 1 | 7.142857 | 56 | 1.1897174 | 0 | 0 | 56 | 0.2508174 + Heap2/MULTI_INSERT | 2 | 14.285714 | 170 | 3.6116421 | 0 | 0 | 170 | 0.76140994 + Heap/INSERT | 2 | 14.285714 | 265 | 5.629913 | 1528 | 8.671964 | 1793 | 8.030636 + Btree/INSERT_LEAF | 5 | 35.714287 | 314 | 6.6709156 | 11820 | 67.08286 | 12134 | 54.346756 + Btree/INSERT_UPPER | 1 | 7.142857 | 61 | 1.2959422 | 136 | 0.77185017 | 197 | 0.8823398 + Btree/SPLIT_L | 1 | 7.142857 | 3749 | 79.64733 | 4136 | 23.473326 | 7885 | 35.315987 +(8 rows) +</screen> + </para> + </listitem> + </varlistentry> + + <varlistentry> + <term> + <function> + pg_get_wal_stats_till_end_of_wal(start_lsn pg_lsn, + per_record boolean DEFAULT false, + "resource_manager/record_type" OUT text, + count OUT int8, + count_percentage OUT float4, + record_length OUT int8, + record_length_percentage OUT float4, + fpi_length OUT int8, + fpi_length_percentage OUT float4, + combined_size OUT int8, + combined_size_percentage OUT float4) + returns setof record + </function> + </term> + + <listitem> + <para> + This function is same as <function>pg_get_wal_stats()</function> except + that it gets statistics of all the valid WAL records from + <replaceable>start_lsn</replaceable> till end of WAL. + </para> + </listitem> + </varlistentry> + + </variablelist> + </sect2> + + <sect2> + <title>Author</title> + + <para> + Bharath Rupireddy <email>[email protected]</email> + </para> + </sect2> + +</sect1> -- 2.25.1 ^ permalink raw reply [nested|flat] 9+ messages in thread
* Re: pg_walinspect - a new extension to get raw WAL data and WAL stats 2022-04-04 03:45 Re: pg_walinspect - a new extension to get raw WAL data and WAL stats Bharath Rupireddy <[email protected]> 2022-04-06 05:02 ` Re: pg_walinspect - a new extension to get raw WAL data and WAL stats Jeff Davis <[email protected]> 2022-04-06 08:45 ` Re: pg_walinspect - a new extension to get raw WAL data and WAL stats Bharath Rupireddy <[email protected]> 2022-04-07 10:05 ` Re: pg_walinspect - a new extension to get raw WAL data and WAL stats Bharath Rupireddy <[email protected]> @ 2022-04-11 10:51 ` Amit Kapila <[email protected]> 2022-04-11 13:03 ` Re: pg_walinspect - a new extension to get raw WAL data and WAL stats Bharath Rupireddy <[email protected]> 0 siblings, 1 reply; 9+ messages in thread From: Amit Kapila @ 2022-04-11 10:51 UTC (permalink / raw) To: Bharath Rupireddy <[email protected]>; +Cc: Jeff Davis <[email protected]>; RKN Sai Krishna <[email protected]>; Andres Freund <[email protected]>; Kyotaro Horiguchi <[email protected]>; Ashutosh Sharma <[email protected]>; Stephen Frost <[email protected]>; Robert Haas <[email protected]>; Andrew Dunstan <[email protected]>; Greg Stark <[email protected]>; Jeremy Schneider <[email protected]>; Bruce Momjian <[email protected]>; PostgreSQL Hackers <[email protected]>; SATYANARAYANA NARLAPURAM <[email protected]>; [email protected]; [email protected] On Thu, Apr 7, 2022 at 3:35 PM Bharath Rupireddy <[email protected]> wrote: > I am facing the below doc build failure on my machine due to this work: ./filelist.sgml:<!ENTITY pgwalinspect SYSTEM "pgwalinspect.sgml"> Tabs appear in SGML/XML files make: *** [check-tabs] Error 1 The attached patch fixes this for me. -- With Regards, Amit Kapila. Attachments: [application/octet-stream] fix_tabs_1.patch (597B, ../../CAA4eK1K6z=gu-jppU1dtsyr2BC-pzrq3TYe=RfY+w386dfdiFA@mail.gmail.com/2-fix_tabs_1.patch) download | inline diff: diff --git a/doc/src/sgml/filelist.sgml b/doc/src/sgml/filelist.sgml index 1e82cb2..40ef5f7 100644 --- a/doc/src/sgml/filelist.sgml +++ b/doc/src/sgml/filelist.sgml @@ -148,7 +148,7 @@ <!ENTITY pgsurgery SYSTEM "pgsurgery.sgml"> <!ENTITY pgtrgm SYSTEM "pgtrgm.sgml"> <!ENTITY pgvisibility SYSTEM "pgvisibility.sgml"> -<!ENTITY pgwalinspect SYSTEM "pgwalinspect.sgml"> +<!ENTITY pgwalinspect SYSTEM "pgwalinspect.sgml"> <!ENTITY postgres-fdw SYSTEM "postgres-fdw.sgml"> <!ENTITY seg SYSTEM "seg.sgml"> <!ENTITY contrib-spi SYSTEM "contrib-spi.sgml"> ^ permalink raw reply [nested|flat] 9+ messages in thread
* Re: pg_walinspect - a new extension to get raw WAL data and WAL stats 2022-04-04 03:45 Re: pg_walinspect - a new extension to get raw WAL data and WAL stats Bharath Rupireddy <[email protected]> 2022-04-06 05:02 ` Re: pg_walinspect - a new extension to get raw WAL data and WAL stats Jeff Davis <[email protected]> 2022-04-06 08:45 ` Re: pg_walinspect - a new extension to get raw WAL data and WAL stats Bharath Rupireddy <[email protected]> 2022-04-07 10:05 ` Re: pg_walinspect - a new extension to get raw WAL data and WAL stats Bharath Rupireddy <[email protected]> 2022-04-11 10:51 ` Re: pg_walinspect - a new extension to get raw WAL data and WAL stats Amit Kapila <[email protected]> @ 2022-04-11 13:03 ` Bharath Rupireddy <[email protected]> 2022-04-12 03:56 ` Re: pg_walinspect - a new extension to get raw WAL data and WAL stats Amit Kapila <[email protected]> 0 siblings, 1 reply; 9+ messages in thread From: Bharath Rupireddy @ 2022-04-11 13:03 UTC (permalink / raw) To: Amit Kapila <[email protected]>; +Cc: Jeff Davis <[email protected]>; RKN Sai Krishna <[email protected]>; Andres Freund <[email protected]>; Kyotaro Horiguchi <[email protected]>; Ashutosh Sharma <[email protected]>; Stephen Frost <[email protected]>; Robert Haas <[email protected]>; Andrew Dunstan <[email protected]>; Greg Stark <[email protected]>; Jeremy Schneider <[email protected]>; Bruce Momjian <[email protected]>; PostgreSQL Hackers <[email protected]>; SATYANARAYANA NARLAPURAM <[email protected]>; [email protected]; [email protected] On Mon, Apr 11, 2022 at 4:21 PM Amit Kapila <[email protected]> wrote: > > On Thu, Apr 7, 2022 at 3:35 PM Bharath Rupireddy > <[email protected]> wrote: > > > > I am facing the below doc build failure on my machine due to this work: > > ./filelist.sgml:<!ENTITY pgwalinspect SYSTEM "pgwalinspect.sgml"> > Tabs appear in SGML/XML files > make: *** [check-tabs] Error 1 > > The attached patch fixes this for me. Thanks. It looks like there's a TAB in between. Your patch LGTM. I'm wondering why this hasn't been caught in the build farm members (or it may have been found but I'm missing to locate it.). Can you please provide me with the doc build command to catch these kinds of errors? Regards, Bharath Rupireddy. ^ permalink raw reply [nested|flat] 9+ messages in thread
* Re: pg_walinspect - a new extension to get raw WAL data and WAL stats 2022-04-04 03:45 Re: pg_walinspect - a new extension to get raw WAL data and WAL stats Bharath Rupireddy <[email protected]> 2022-04-06 05:02 ` Re: pg_walinspect - a new extension to get raw WAL data and WAL stats Jeff Davis <[email protected]> 2022-04-06 08:45 ` Re: pg_walinspect - a new extension to get raw WAL data and WAL stats Bharath Rupireddy <[email protected]> 2022-04-07 10:05 ` Re: pg_walinspect - a new extension to get raw WAL data and WAL stats Bharath Rupireddy <[email protected]> 2022-04-11 10:51 ` Re: pg_walinspect - a new extension to get raw WAL data and WAL stats Amit Kapila <[email protected]> 2022-04-11 13:03 ` Re: pg_walinspect - a new extension to get raw WAL data and WAL stats Bharath Rupireddy <[email protected]> @ 2022-04-12 03:56 ` Amit Kapila <[email protected]> 0 siblings, 0 replies; 9+ messages in thread From: Amit Kapila @ 2022-04-12 03:56 UTC (permalink / raw) To: Bharath Rupireddy <[email protected]>; +Cc: Jeff Davis <[email protected]>; RKN Sai Krishna <[email protected]>; Andres Freund <[email protected]>; Kyotaro Horiguchi <[email protected]>; Ashutosh Sharma <[email protected]>; Stephen Frost <[email protected]>; Robert Haas <[email protected]>; Andrew Dunstan <[email protected]>; Greg Stark <[email protected]>; Jeremy Schneider <[email protected]>; Bruce Momjian <[email protected]>; PostgreSQL Hackers <[email protected]>; SATYANARAYANA NARLAPURAM <[email protected]>; [email protected]; [email protected] On Mon, Apr 11, 2022 at 6:33 PM Bharath Rupireddy <[email protected]> wrote: > > On Mon, Apr 11, 2022 at 4:21 PM Amit Kapila <[email protected]> wrote: > > > > On Thu, Apr 7, 2022 at 3:35 PM Bharath Rupireddy > > <[email protected]> wrote: > > > > > > > I am facing the below doc build failure on my machine due to this work: > > > > ./filelist.sgml:<!ENTITY pgwalinspect SYSTEM "pgwalinspect.sgml"> > > Tabs appear in SGML/XML files > > make: *** [check-tabs] Error 1 > > > > The attached patch fixes this for me. > > Thanks. It looks like there's a TAB in between. Your patch LGTM. > > I'm wondering why this hasn't been caught in the build farm members > (or it may have been found but I'm missing to locate it.). > > Can you please provide me with the doc build command to catch these > kinds of errors? > Nothing special. In the doc/src/sgml, I did make clean followed by make check. -- With Regards, Amit Kapila. ^ permalink raw reply [nested|flat] 9+ messages in thread
* [PATCH v4 2/7] Row pattern recognition patch (parse/analysis). @ 2023-08-09 07:56 Tatsuo Ishii <[email protected]> 0 siblings, 0 replies; 9+ messages in thread From: Tatsuo Ishii @ 2023-08-09 07:56 UTC (permalink / raw) --- src/backend/parser/parse_agg.c | 7 + src/backend/parser/parse_clause.c | 292 +++++++++++++++++++++++++++++- src/backend/parser/parse_expr.c | 4 + src/backend/parser/parse_func.c | 3 + 4 files changed, 305 insertions(+), 1 deletion(-) diff --git a/src/backend/parser/parse_agg.c b/src/backend/parser/parse_agg.c index 85cd47b7ae..aa7a1cee80 100644 --- a/src/backend/parser/parse_agg.c +++ b/src/backend/parser/parse_agg.c @@ -564,6 +564,10 @@ check_agglevels_and_constraints(ParseState *pstate, Node *expr) errkind = true; break; + case EXPR_KIND_RPR_DEFINE: + errkind = true; + break; + /* * There is intentionally no default: case here, so that the * compiler will warn if we add a new ParseExprKind without @@ -953,6 +957,9 @@ transformWindowFuncCall(ParseState *pstate, WindowFunc *wfunc, case EXPR_KIND_CYCLE_MARK: errkind = true; break; + case EXPR_KIND_RPR_DEFINE: + errkind = true; + break; /* * There is intentionally no default: case here, so that the diff --git a/src/backend/parser/parse_clause.c b/src/backend/parser/parse_clause.c index 334b9b42bd..60020a7025 100644 --- a/src/backend/parser/parse_clause.c +++ b/src/backend/parser/parse_clause.c @@ -100,7 +100,10 @@ static WindowClause *findWindowClause(List *wclist, const char *name); static Node *transformFrameOffset(ParseState *pstate, int frameOptions, Oid rangeopfamily, Oid rangeopcintype, Oid *inRangeFunc, Node *clause); - +static void transformRPR(ParseState *pstate, WindowClause *wc, WindowDef *windef, List **targetlist); +static List *transformDefineClause(ParseState *pstate, WindowClause *wc, WindowDef *windef, List **targetlist); +static void transformPatternClause(ParseState *pstate, WindowClause *wc, WindowDef *windef); +static List *transformMeasureClause(ParseState *pstate, WindowClause *wc, WindowDef *windef); /* * transformFromClause - @@ -2950,6 +2953,10 @@ transformWindowDefinitions(ParseState *pstate, rangeopfamily, rangeopcintype, &wc->endInRangeFunc, windef->endOffset); + + /* Process Row Pattern Recognition related clauses */ + transformRPR(pstate, wc, windef, targetlist); + wc->runCondition = NIL; wc->winref = winref; @@ -3815,3 +3822,286 @@ transformFrameOffset(ParseState *pstate, int frameOptions, return node; } + +/* + * transformRPR + * Process Row Pattern Recognition related clauses + */ +static void +transformRPR(ParseState *pstate, WindowClause *wc, WindowDef *windef, List **targetlist) +{ + /* + * Window definition exists? + */ + if (windef == NULL) + return; + + /* + * Row Pattern Common Syntax clause exists? + */ + if (windef->rpCommonSyntax == NULL) + return; + + /* Check Frame option. Frame must start at current row */ + if ((wc->frameOptions & FRAMEOPTION_START_CURRENT_ROW) == 0) + ereport(ERROR, + (errcode(ERRCODE_SYNTAX_ERROR), + errmsg("FRAME must start at current row when row patttern recognition is used"))); + + /* Transform AFTER MACH SKIP TO clause */ + wc->rpSkipTo = windef->rpCommonSyntax->rpSkipTo; + + /* Transform SEEK or INITIAL clause */ + wc->initial = windef->rpCommonSyntax->initial; + + /* Transform DEFINE clause into list of TargetEntry's */ + wc->defineClause = transformDefineClause(pstate, wc, windef, targetlist); + + /* Check PATTERN clause and copy to patternClause */ + transformPatternClause(pstate, wc, windef); + + /* Transform MEASURE clause */ + transformMeasureClause(pstate, wc, windef); +} + +/* + * transformDefineClause Process DEFINE clause and transform ResTarget into + * list of TargetEntry. + * + * XXX we only support column reference in row pattern definition search + * condition, e.g. "price". <row pattern definition variable name>.<column + * reference> is not supported, e.g. "A.price". + */ +static List * +transformDefineClause(ParseState *pstate, WindowClause *wc, WindowDef *windef, List **targetlist) +{ + /* DEFINE variable name initials */ + static char *defineVariableInitials = "abcdefghijklmnopqrstuvwxyz"; + + ListCell *lc, *l; + ResTarget *restarget, *r; + List *restargets; + char *name; + int initialLen; + int i; + + /* + * If Row Definition Common Syntax exists, DEFINE clause must exist. + * (the raw parser should have already checked it.) + */ + Assert(windef->rpCommonSyntax->rpDefs != NULL); + + /* + * Check and add "A AS A IS TRUE" if pattern variable is missing in DEFINE + * per the SQL standard. + */ + restargets = NIL; + foreach(lc, windef->rpCommonSyntax->rpPatterns) + { + A_Expr *a; + bool found = false; + + if (!IsA(lfirst(lc), A_Expr)) + ereport(ERROR, + errmsg("node type is not A_Expr")); + + a = (A_Expr *)lfirst(lc); + name = strVal(a->lexpr); + + foreach(l, windef->rpCommonSyntax->rpDefs) + { + restarget = (ResTarget *)lfirst(l); + + if (!strcmp(restarget->name, name)) + { + found = true; + break; + } + } + + if (!found) + { + /* + * "name" is missing. So create "name AS name IS TRUE" ResTarget + * node and add it to the temporary list. + */ + A_Const *n; + + restarget = makeNode(ResTarget); + n = makeNode(A_Const); + n->val.boolval.type = T_Boolean; + n->val.boolval.boolval = true; + n->location = -1; + restarget->name = pstrdup(name); + restarget->indirection = NIL; + restarget->val = (Node *)n; + restarget->location = -1; + restargets = lappend((List *)restargets, restarget); + } + } + + if (list_length(restargets) >= 1) + { + /* add missing DEFINEs */ + windef->rpCommonSyntax->rpDefs = list_concat(windef->rpCommonSyntax->rpDefs, + restargets); + list_free(restargets); + } + + /* + * Check for duplicate row pattern definition variables. The standard + * requires that no two row pattern definition variable names shall be + * equivalent. + */ + restargets = NIL; + foreach(lc, windef->rpCommonSyntax->rpDefs) + { + restarget = (ResTarget *)lfirst(lc); + name = restarget->name; + + /* + * Add DEFINE expression (Restarget->val) to the targetlist as a + * TargetEntry if it does not exist yet. Planner will add the column + * ref var node to the outer plan's target list later on. This makes + * DEFINE expression could access the outer tuple while evaluating + * PATTERN. + * + * XXX: adding whole expressions of DEFINE to the plan.targetlist is + * not so good, because it's not necessary to evalute the expression + * in the target list while running the plan. We should extract the + * var nodes only then add them to the plan.targetlist. + */ + findTargetlistEntrySQL99(pstate, (Node *)restarget->val, targetlist, EXPR_KIND_RPR_DEFINE); + + /* + * Make sure that the row pattern definition search condition is a + * boolean expression. + */ + transformWhereClause(pstate, restarget->val, + EXPR_KIND_RPR_DEFINE, "DEFINE"); + + foreach(l, restargets) + { + char *n; + + r = (ResTarget *) lfirst(l); + n = r->name; + + if (!strcmp(n, name)) + ereport(ERROR, + (errcode(ERRCODE_SYNTAX_ERROR), + errmsg("row pattern definition variable name \"%s\" appears more than once in DEFINE clause", + name), + parser_errposition(pstate, exprLocation((Node *)r)))); + } + restargets = lappend(restargets, restarget); + } + list_free(restargets); + + /* + * Create list of row pattern DEFINE variable name's initial. + * We assign [a-z] to them (up to 26 variable names are allowed). + */ + restargets = NIL; + i = 0; + initialLen = strlen(defineVariableInitials); + + foreach(lc, windef->rpCommonSyntax->rpDefs) + { + char initial[2]; + + restarget = (ResTarget *)lfirst(lc); + name = restarget->name; + + if (i >= initialLen) + { + ereport(ERROR, + (errcode(ERRCODE_SYNTAX_ERROR), + errmsg("number of row pattern definition variable names exceeds %d", initialLen), + parser_errposition(pstate, exprLocation((Node *)restarget)))); + } + initial[0] = defineVariableInitials[i++]; + initial[1] = '\0'; + wc->defineInitial = lappend(wc->defineInitial, makeString(pstrdup(initial))); + } + + return transformTargetList(pstate, windef->rpCommonSyntax->rpDefs, + EXPR_KIND_RPR_DEFINE); +} + +/* + * transformPatternClause + * Process PATTERN clause and return PATTERN clause in the raw parse tree + */ +static void +transformPatternClause(ParseState *pstate, WindowClause *wc, WindowDef *windef) +{ + ListCell *lc, *l; + + /* + * Row Pattern Common Syntax clause exists? + */ + if (windef->rpCommonSyntax == NULL) + return; + + /* + * Primary row pattern variable names in PATTERN clause must appear in + * DEFINE clause as row pattern definition variable names. + */ + wc->patternVariable = NIL; + wc->patternRegexp = NIL; + foreach(lc, windef->rpCommonSyntax->rpPatterns) + { + A_Expr *a; + char *name; + char *regexp; + bool found = false; + + if (!IsA(lfirst(lc), A_Expr)) + ereport(ERROR, + errmsg("node type is not A_Expr")); + + a = (A_Expr *)lfirst(lc); + name = strVal(a->lexpr); + + foreach(l, windef->rpCommonSyntax->rpDefs) + { + ResTarget *restarget = (ResTarget *)lfirst(l); + + if (!strcmp(restarget->name, name)) + { + found = true; + break; + } + } + + if (!found) + { + ereport(ERROR, + (errcode(ERRCODE_SYNTAX_ERROR), + errmsg("primary row pattern variable name \"%s\" does not appear in DEFINE clause", + name), + parser_errposition(pstate, exprLocation((Node *)a)))); + } + wc->patternVariable = lappend(wc->patternVariable, makeString(pstrdup(name))); + regexp = strVal(lfirst(list_head(a->name))); + wc->patternRegexp = lappend(wc->patternRegexp, makeString(pstrdup(regexp))); + } +} + +/* + * transformMeasureClause + * Process MEASURE clause + * XXX MEASURE clause is not supported yet + */ +static List * +transformMeasureClause(ParseState *pstate, WindowClause *wc, WindowDef *windef) +{ + if (windef->rowPatternMeasures == NIL) + return NIL; + + ereport(ERROR, + (errcode(ERRCODE_SYNTAX_ERROR), + errmsg("%s","MEASURE clause is not supported yet"), + parser_errposition(pstate, exprLocation((Node *)windef->rowPatternMeasures)))); +} diff --git a/src/backend/parser/parse_expr.c b/src/backend/parser/parse_expr.c index fed8e4d089..8921b7ae01 100644 --- a/src/backend/parser/parse_expr.c +++ b/src/backend/parser/parse_expr.c @@ -557,6 +557,7 @@ transformColumnRef(ParseState *pstate, ColumnRef *cref) case EXPR_KIND_COPY_WHERE: case EXPR_KIND_GENERATED_COLUMN: case EXPR_KIND_CYCLE_MARK: + case EXPR_KIND_RPR_DEFINE: /* okay */ break; @@ -1770,6 +1771,7 @@ transformSubLink(ParseState *pstate, SubLink *sublink) case EXPR_KIND_VALUES: case EXPR_KIND_VALUES_SINGLE: case EXPR_KIND_CYCLE_MARK: + case EXPR_KIND_RPR_DEFINE: /* okay */ break; case EXPR_KIND_CHECK_CONSTRAINT: @@ -3149,6 +3151,8 @@ ParseExprKindName(ParseExprKind exprKind) return "GENERATED AS"; case EXPR_KIND_CYCLE_MARK: return "CYCLE"; + case EXPR_KIND_RPR_DEFINE: + return "DEFINE"; /* * There is intentionally no default: case here, so that the diff --git a/src/backend/parser/parse_func.c b/src/backend/parser/parse_func.c index b3f0b6a137..2ff3699538 100644 --- a/src/backend/parser/parse_func.c +++ b/src/backend/parser/parse_func.c @@ -2656,6 +2656,9 @@ check_srf_call_placement(ParseState *pstate, Node *last_srf, int location) case EXPR_KIND_CYCLE_MARK: errkind = true; break; + case EXPR_KIND_RPR_DEFINE: + errkind = true; + break; /* * There is intentionally no default: case here, so that the -- 2.25.1 ----Next_Part(Wed_Aug__9_17_41_12_2023_134)-- Content-Type: Text/X-Patch; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="v4-0003-Row-pattern-recognition-patch-planner.patch" ^ permalink raw reply [nested|flat] 9+ messages in thread
end of thread, other threads:[~2023-08-09 07:56 UTC | newest] Thread overview: 9+ messages (download: mbox mbox.gz follow: Atom feed) -- links below jump to the message on this page -- 2017-12-21 12:23 [PATCH 2/6] Add monitoring aid for max_slot_wal_keep_size Kyotaro Horiguchi <[email protected]> 2022-04-04 03:45 Re: pg_walinspect - a new extension to get raw WAL data and WAL stats Bharath Rupireddy <[email protected]> 2022-04-06 05:02 ` Re: pg_walinspect - a new extension to get raw WAL data and WAL stats Jeff Davis <[email protected]> 2022-04-06 08:45 ` Re: pg_walinspect - a new extension to get raw WAL data and WAL stats Bharath Rupireddy <[email protected]> 2022-04-07 10:05 ` Re: pg_walinspect - a new extension to get raw WAL data and WAL stats Bharath Rupireddy <[email protected]> 2022-04-11 10:51 ` Re: pg_walinspect - a new extension to get raw WAL data and WAL stats Amit Kapila <[email protected]> 2022-04-11 13:03 ` Re: pg_walinspect - a new extension to get raw WAL data and WAL stats Bharath Rupireddy <[email protected]> 2022-04-12 03:56 ` Re: pg_walinspect - a new extension to get raw WAL data and WAL stats Amit Kapila <[email protected]> 2023-08-09 07:56 [PATCH v4 2/7] Row pattern recognition patch (parse/analysis). Tatsuo Ishii <[email protected]>
This inbox is served by agora; see mirroring instructions for how to clone and mirror all data and code used for this inbox