pg.ddx.io pgsql-hackers@postgresql.org mailing list archive help / color / mirror / Atom feed From: movead.li@highgo.ca <movead.li@highgo.ca>
To: Shinya11.Kato@nttdata.com <Shinya11.Kato@nttdata.com>
Cc: pgsql-hackers <pgsql-hackers@postgresql.org>
Cc: andres <andres@anarazel.de>
Cc: michael <michael@paquier.xyz>
Cc: ahsan.hadi <ahsan.hadi@highgo.ca>
Cc: Horiguchi <horikyota.ntt@gmail.com>
Subject: RE: Wrong statistics for size of XLOG_SWITCH during pg_waldump.
Date: Wed, 6 Jan 2021 11:14:37 +0800
Message-ID: <2021010611143203905334@highgo.ca> (raw )
References: <20201016.180033.217520066292592870.horikyota.ntt@gmail.com >
<2020101617395691465546@highgo.ca >
<240cd11dc6664dfcbef50c2d6b16ccb2@MP-MSGSS-MBX001.msg.nttdata.co.jp >
<20201204.152047.1606427486743708152.horikyota.ntt@gmail.com >
<b0a32c58414546b78b03f2c281f2f145@MP-MSGSS-MBX001.msg.nttdata.co.jp >
Thanks for review, and sorry for reply so later.
>I reviewed the patch and found some problems.
>>+ if(startSegNo != endSegNo)
>>+ else if(record->ReadRecPtr / XLOG_BLCKSZ !=
>>+ if(rmid == RM_XLOG_ID && info == XLOG_SWITCH)
>>+ if(ri == RM_XLOG_ID)
>>+ if(info == XLOG_SWITCH)
>You need to put a space after the "if".
All fix and thanks for point the issue.
>>@@ -24,6 +24,7 @@
>>#include "common/logging.h"
>>#include "getopt_long.h"
>>#include "rmgrdesc.h"
>>+#include "catalog/pg_control.h"
>I think the include statements should be arranged in alphabetical order.
Fix.
>>+ info = (rj << 4) & ~XLR_INFO_MASK;
>>+ if(info == XLOG_SWITCH)
>>+ XLogDumpStatsRow(psprintf("XLOG/SWITCH_JUNK"),
>>+ 0, total_count, stats->junk_size, total_rec_len,
>>+ 0, total_fpi_len, stats->junk_size, total_len);
>Can't be described in the same way as "XLogDumpStatsRow(psprintf("%s/%s", desc->rm_name, id)..."?
>Only this part looks strange.
>Why are the "count" and "fpi_len" fields 0?
The 'SWITCH_JUNK' is not a real record and it relys on 'XLOG_SWITCH' record, so I think we can't count
'SWITCH_JUNK', so the "count" is 0. And it never contain FPI, so the "fpi_len" is 0.
But 0 value maybe looks strange, so in current version I show it like below:
Type N (%) Record size (%) FPI size (%) Combined size (%)
---- - --- ----------- --- -------- --- ------------- ---
...
XLOG/SWITCH_JUNK - ( -) 11006248 ( 72.26) - ( -) 11006248 ( 65.78)
Transaction/COMMIT 10 ( 0.03) 340 ( 0.00) 0 ( 0.00) 340 ( 0.00)
>I think you need to improve the duplicate output in column "XLOG/SWITCH_JUNK".
Yes it's a bug and fixed.
Regards,
Highgo Software (Canada/China/Pakistan)
URL : www.highgo.ca
EMAIL: mailto:movead(dot)li(at)highgo(dot)ca
Attachments:
fix_waldump_size_for_wal_switch_v6.patch (application/octet-stream, 5.4K) download
diff --git a/src/backend/access/rmgrdesc/xlogdesc.c b/src/backend/access/rmgrdesc/xlogdesc.c
index 92cc7ea073..0668511660 100644
--- a/src/backend/access/rmgrdesc/xlogdesc.c
+++ b/src/backend/access/rmgrdesc/xlogdesc.c
@@ -74,6 +74,13 @@ xlog_desc(StringInfo buf, XLogReaderState *record)
memcpy(&nextOid, rec, sizeof(Oid));
appendStringInfo(buf, "%u", nextOid);
}
+ else if (info == XLOG_SWITCH)
+ {
+ uint32 junk_len;
+ junk_len = xlog_switch_junk_len(record);
+
+ appendStringInfo(buf, "trailing-bytes: %u", junk_len);
+ }
else if (info == XLOG_RESTORE_POINT)
{
xl_restore_point *xlrec = (xl_restore_point *) rec;
@@ -142,6 +149,33 @@ xlog_desc(StringInfo buf, XLogReaderState *record)
}
}
+uint32
+xlog_switch_junk_len(XLogReaderState *record)
+{
+ uint32 junk_len;
+ XLogSegNo startSegNo;
+ XLogSegNo endSegNo;
+
+ XLByteToSeg(record->ReadRecPtr, startSegNo, record->segcxt.ws_segsize);
+ XLByteToPrevSeg(record->EndRecPtr, endSegNo, record->segcxt.ws_segsize);
+
+ junk_len = record->EndRecPtr - record->ReadRecPtr -
+ XLogRecGetTotalLen(record);
+ /*
+ * If the wal switch record spread on two pages, we should extra minus
+ * the page head, be careful if it's page at the top of a wal segment.
+ */
+ if (startSegNo != endSegNo)
+ junk_len -= SizeOfXLogLongPHD;
+ else if (record->ReadRecPtr / XLOG_BLCKSZ !=
+ (record->EndRecPtr - 1) / XLOG_BLCKSZ)
+ junk_len -= SizeOfXLogShortPHD;
+
+ Assert(junk_len >= 0);
+
+ return junk_len;
+}
+
const char *
xlog_identify(uint8 info)
{
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index 164868d16e..c20f7f5703 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -20,6 +20,7 @@
#include "access/xlog_internal.h"
#include "access/xlogreader.h"
#include "access/xlogrecord.h"
+#include "catalog/pg_control.h"
#include "common/fe_memutils.h"
#include "common/logging.h"
#include "getopt_long.h"
@@ -66,6 +67,7 @@ typedef struct Stats
typedef struct XLogDumpStats
{
uint64 count;
+ uint32 junk_size;
Stats rmgr_stats[RM_NEXT_ID];
Stats record_stats[RM_NEXT_ID][MAX_XLINFO_TYPES];
} XLogDumpStats;
@@ -416,12 +418,21 @@ XLogDumpCountRecord(XLogDumpConfig *config, XLogDumpStats *stats,
uint8 recid;
uint32 rec_len;
uint32 fpi_len;
+ uint8 info;
stats->count++;
rmid = XLogRecGetRmid(record);
+ info = XLogRecGetInfo(record) & ~XLR_INFO_MASK;
XLogDumpRecordLen(record, &rec_len, &fpi_len);
+ /* Add junk-space stats for XLOG_SWITCH */
+ if (rmid == RM_XLOG_ID && info == XLOG_SWITCH)
+ {
+ uint32 junk_len = xlog_switch_junk_len(record);
+
+ stats->junk_size += junk_len;
+ }
/* Update per-rmgr statistics */
@@ -595,6 +606,38 @@ XLogDumpStatsRow(const char *name,
tot_len, tot_len_pct);
}
+static void
+XLogDumpStatsRow_junk(bool detail,
+ uint64 rec_len, uint64 total_rec_len,
+ uint64 tot_len, uint64 total_len)
+{
+ double rec_len_pct,
+ tot_len_pct;
+ char *name = NULL;
+ char pad = '-';
+
+ if (detail)
+ name = "XLOG/SWITCH_JUNK";
+ else
+ name = "XLOGSwitchJunk";
+
+ rec_len_pct = 0;
+ if (total_rec_len != 0)
+ rec_len_pct = 100 * (double) rec_len / total_rec_len;
+
+ tot_len_pct = 0;
+ if (total_len != 0)
+ tot_len_pct = 100 * (double) tot_len / total_len;
+
+ printf("%-27s "
+ "%20c (%6c) "
+ "%20" INT64_MODIFIER "u (%6.02f) "
+ "%20c (%6c) "
+ "%20" INT64_MODIFIER "u (%6.02f)\n",
+ name, pad, pad, rec_len, rec_len_pct, pad, pad,
+ tot_len, tot_len_pct);
+}
+
/*
* Display summary statistics about the records seen so far.
@@ -622,6 +665,7 @@ XLogDumpDisplayStats(XLogDumpConfig *config, XLogDumpStats *stats)
total_rec_len += stats->rmgr_stats[ri].rec_len;
total_fpi_len += stats->rmgr_stats[ri].fpi_len;
}
+ total_rec_len += stats->junk_size;
total_len = total_rec_len + total_fpi_len;
/*
@@ -652,12 +696,22 @@ XLogDumpDisplayStats(XLogDumpConfig *config, XLogDumpStats *stats)
XLogDumpStatsRow(desc->rm_name,
count, total_count, rec_len, total_rec_len,
fpi_len, total_fpi_len, tot_len, total_len);
+ if (ri == RM_XLOG_ID)
+ {
+ XLogDumpStatsRow_junk(false,
+ stats->junk_size, total_rec_len,
+ stats->junk_size, total_len);
+ }
+
}
else
{
+ bool getswitch = false;
+
for (rj = 0; rj < MAX_XLINFO_TYPES; rj++)
{
const char *id;
+ uint8 info;
count = stats->record_stats[ri][rj].count;
rec_len = stats->record_stats[ri][rj].rec_len;
@@ -676,7 +730,15 @@ XLogDumpDisplayStats(XLogDumpConfig *config, XLogDumpStats *stats)
XLogDumpStatsRow(psprintf("%s/%s", desc->rm_name, id),
count, total_count, rec_len, total_rec_len,
fpi_len, total_fpi_len, tot_len, total_len);
+ info = (rj << 4) & ~XLR_INFO_MASK;
+ if (ri == RM_XLOG_ID && info == XLOG_SWITCH)
+ getswitch = true;
+
}
+ if (getswitch)
+ XLogDumpStatsRow_junk(true,
+ stats->junk_size, total_rec_len,
+ stats->junk_size, total_len);
}
}
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index 9585ad17b3..687ba90238 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -315,6 +315,7 @@ extern XLogRecPtr RequestXLogSwitch(bool mark_unimportant);
extern void GetOldestRestartPoint(XLogRecPtr *oldrecptr, TimeLineID *oldtli);
+extern uint32 xlog_switch_junk_len(XLogReaderState *record);
/*
* Exported for the functions in timeline.c and xlogarchive.c. Only valid
* in the startup process.
view thread (28+ messages) latest in thread
Message-ID: <2021010611143203905334@highgo.ca>
Permalink: ../2021010611143203905334@highgo.ca/
Also on: postgresql.org/message-id/2021010611143203905334@highgo.ca
copy link · copy postgr.es · copy git am
reply Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Reply to all the recipients using the --to and --cc options:
reply via email
To: pgsql-hackers@postgresql.org
Cc: movead.li@highgo.ca, Shinya11.Kato@nttdata.com, andres@anarazel.de, michael@paquier.xyz, ahsan.hadi@highgo.ca, horikyota.ntt@gmail.com
Subject: RE: Wrong statistics for size of XLOG_SWITCH during pg_waldump.
In-Reply-To: <2021010611143203905334@highgo.ca>
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
This inbox is served by DDX for PostgreSQL; see mirroring instructions
for how to clone and mirror all data and code used for this inbox