agora inbox for [email protected]
help / color / mirror / Atom feedFrom: Kyotaro Horiguchi <[email protected]>
Subject: [PATCH 1/4] Add WAL releaf vent for replication slots
Date: Thu, 21 Dec 2017 21:20:20 +0900
Adds a capability to limit the number of segments kept by replication
slots by a GUC variable.
---
src/backend/access/transam/xlog.c | 114 +++++++++++++++++++++-----
src/backend/utils/misc/guc.c | 11 +++
src/backend/utils/misc/postgresql.conf.sample | 1 +
src/include/access/xlog.h | 1 +
4 files changed, 107 insertions(+), 20 deletions(-)
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index e42b828..55694cb 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -105,6 +105,7 @@ int wal_level = WAL_LEVEL_MINIMAL;
int CommitDelay = 0; /* precommit delay in microseconds */
int CommitSiblings = 5; /* # concurrent xacts needed to sleep */
int wal_retrieve_retry_interval = 5000;
+int max_slot_wal_keep_size_mb = 0;
#ifdef WAL_DEBUG
bool XLOG_DEBUG = false;
@@ -861,6 +862,7 @@ static void checkTimeLineSwitch(XLogRecPtr lsn, TimeLineID newTLI,
static void LocalSetXLogInsertAllowed(void);
static void CreateEndOfRecoveryRecord(void);
static void CheckPointGuts(XLogRecPtr checkPointRedo, int flags);
+static XLogSegNo GetMinKeepSegment(XLogRecPtr currpos, XLogRecPtr minSlotPtr);
static void KeepLogSeg(XLogRecPtr recptr, XLogSegNo *logSegNo);
static XLogRecPtr XLogGetReplicationSlotMinimumLSN(void);
@@ -9348,6 +9350,74 @@ CreateRestartPoint(int flags)
}
/*
+ * Returns minimum segment number the next checktpoint must leave considering
+ * wal_keep_segments, replication slots and max_slot_wal_keep_size.
+ */
+static XLogSegNo
+GetMinKeepSegment(XLogRecPtr currpos, XLogRecPtr minSlotPtr)
+{
+ uint64 keepSegs;
+ XLogSegNo currSeg;
+ XLogSegNo tailSeg;
+ uint64 slotlimitbytes;
+ uint64 slotlimitfragment;
+ uint64 currposoff;
+ XLogRecPtr slotpos = minSlotPtr;
+ XLogSegNo slotSeg;
+
+ Assert(wal_keep_segments >= 0);
+ Assert(max_slot_wal_keep_size_mb >= 0);
+
+ XLByteToSeg(currpos, currSeg, wal_segment_size);
+ XLByteToSeg(slotpos, slotSeg, wal_segment_size);
+
+ /*
+ * wal_keep_segments keeps more segments than slot, slotpos is no longer
+ * useful. Don't perform subtraction to keep values positive.
+ */
+ if (slotpos != InvalidXLogRecPtr && currSeg <= slotSeg + wal_keep_segments)
+ slotpos = InvalidXLogRecPtr;
+
+ /* slots aren't useful, consider only wal_keep_segments */
+ if (slotpos == InvalidXLogRecPtr)
+ {
+ /* avoid underflow, don't go below 1 */
+ if (currSeg <= wal_keep_segments)
+ return 1;
+
+ return currSeg - wal_keep_segments;
+ }
+
+ /* just return slotSeg if we don't put a limit */
+ if (max_slot_wal_keep_size_mb == 0)
+ return slotSeg;
+
+ /*
+ * Slot limit is defined and slot gives the oldest segment to keep,
+ * calculate the oldest segment that should not be removed
+ */
+ slotlimitbytes = 1024 * 1024 * max_slot_wal_keep_size_mb;
+ slotlimitfragment = XLogSegmentOffset(slotlimitbytes,
+ wal_segment_size);
+ currposoff = XLogSegmentOffset(currpos, wal_segment_size);
+ keepSegs = wal_keep_segments +
+ ConvertToXSegs(max_slot_wal_keep_size_mb, wal_segment_size);
+ if (currposoff < slotlimitfragment)
+ keepSegs++;
+
+ /*
+ * calculate the oldest segment that is kept by wal_keep_segments and
+ * max_slot_wal_keep_size.
+ */
+ if (currSeg <= keepSegs)
+ tailSeg = 1;
+ else
+ tailSeg = currSeg - keepSegs;
+
+ return tailSeg;
+}
+
+/*
* Retreat *logSegNo to the last segment that we need to retain because of
* either wal_keep_segments or replication slots.
*
@@ -9359,34 +9429,38 @@ static void
KeepLogSeg(XLogRecPtr recptr, XLogSegNo *logSegNo)
{
XLogSegNo segno;
- XLogRecPtr keep;
+ XLogRecPtr slotminptr = InvalidXLogRecPtr;
+ XLogSegNo minSegNo;
+ XLogSegNo slotSegNo;
XLByteToSeg(recptr, segno, wal_segment_size);
- keep = XLogGetReplicationSlotMinimumLSN();
- /* compute limit for wal_keep_segments first */
- if (wal_keep_segments > 0)
- {
- /* avoid underflow, don't go below 1 */
- if (segno <= wal_keep_segments)
- segno = 1;
- else
- segno = segno - wal_keep_segments;
- }
+ if (max_replication_slots > 0)
+ slotminptr = XLogGetReplicationSlotMinimumLSN();
- /* then check whether slots limit removal further */
- if (max_replication_slots > 0 && keep != InvalidXLogRecPtr)
- {
- XLogSegNo slotSegNo;
+ /*
+ * We should keep certain number of WAL segments after this checktpoint.
+ */
+ minSegNo = GetMinKeepSegment(recptr, slotminptr);
- XLByteToSeg(keep, slotSegNo, wal_segment_size);
+ /*
+ * warn if the checkpoint flushes the segments required by replication
+ * slots.
+ */
+ if (!XLogRecPtrIsInvalid(slotminptr))
+ {
+ XLByteToSeg(slotminptr, slotSegNo, wal_segment_size);
- if (slotSegNo <= 0)
- segno = 1;
- else if (slotSegNo < segno)
- segno = slotSegNo;
+ if (slotSegNo < minSegNo)
+ ereport(WARNING,
+ (errmsg ("some replication slots have lost required WAL segments"),
+ errdetail("The mostly affected slot has lost %ld segments.",
+ minSegNo - slotSegNo)));
}
+ if (minSegNo < segno)
+ segno = minSegNo;
+
/* don't delete WAL segments newer than the calculated segment */
if (segno < *logSegNo)
*logSegNo = segno;
diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c
index 5884fa9..0eb3f46 100644
--- a/src/backend/utils/misc/guc.c
+++ b/src/backend/utils/misc/guc.c
@@ -2397,6 +2397,17 @@ static struct config_int ConfigureNamesInt[] =
},
{
+ {"max_slot_wal_keep_size", PGC_SIGHUP, REPLICATION_SENDING,
+ gettext_noop("Sets the maximum size of extra WALs kept by replication slots."),
+ NULL,
+ GUC_UNIT_MB
+ },
+ &max_slot_wal_keep_size_mb,
+ 0, 0, INT_MAX,
+ NULL, NULL, NULL
+ },
+
+ {
{"wal_sender_timeout", PGC_SIGHUP, REPLICATION_SENDING,
gettext_noop("Sets the maximum time to wait for WAL replication."),
NULL,
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index abffde6..db4ae2b 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -236,6 +236,7 @@
#max_wal_senders = 10 # max number of walsender processes
# (change requires restart)
#wal_keep_segments = 0 # in logfile segments; 0 disables
+#max_slot_wal_keep_size = 0 # measured in bytes; 0 disables
#wal_sender_timeout = 60s # in milliseconds; 0 disables
#max_replication_slots = 10 # max number of replication slots
diff --git a/src/include/access/xlog.h b/src/include/access/xlog.h
index 421ba6d..12cd0d1 100644
--- a/src/include/access/xlog.h
+++ b/src/include/access/xlog.h
@@ -98,6 +98,7 @@ extern int wal_segment_size;
extern int min_wal_size_mb;
extern int max_wal_size_mb;
extern int wal_keep_segments;
+extern int max_slot_wal_keep_size_mb;
extern int XLOGbuffers;
extern int XLogArchiveTimeout;
extern int wal_retrieve_retry_interval;
--
2.9.2
----Next_Part(Mon_Jan_29_19_26_34_2018_950)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
filename="0002-Add-monitoring-aid-for-max_replication_slots.patch"
view thread (37+ messages) latest in thread
reply
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Reply to all the recipients using the --to and --cc options:
reply via email
To: [email protected]
Cc: [email protected]
Subject: Re: [PATCH 1/4] Add WAL releaf vent for replication slots
In-Reply-To: <no-message-id-46022@localhost>
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
This inbox is served by agora; see mirroring instructions
for how to clone and mirror all data and code used for this inbox