agora inbox for pgsql-hackers@postgresql.org
help / color / mirror / Atom feedFrom: Kyotaro Horiguchi <horiguchi.kyotaro@lab.ntt.co.jp>
Subject: [PATCH 1/4] Add WAL releaf vent for replication slots
Date: Thu, 21 Dec 2017 21:20:20 +0900
Adds a capability to limit the number of segments kept by replication
slots by a GUC variable.
---
src/backend/access/transam/xlog.c | 100 ++++++++++++++++++++------
src/backend/utils/misc/guc.c | 12 ++++
src/backend/utils/misc/postgresql.conf.sample | 1 +
src/include/access/xlog.h | 1 +
4 files changed, 94 insertions(+), 20 deletions(-)
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 44017d33e4..c27a589e89 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -105,6 +105,7 @@ int wal_level = WAL_LEVEL_MINIMAL;
int CommitDelay = 0; /* precommit delay in microseconds */
int CommitSiblings = 5; /* # concurrent xacts needed to sleep */
int wal_retrieve_retry_interval = 5000;
+int max_slot_wal_keep_size_mb = 0;
#ifdef WAL_DEBUG
bool XLOG_DEBUG = false;
@@ -867,6 +868,7 @@ static void checkTimeLineSwitch(XLogRecPtr lsn, TimeLineID newTLI,
static void LocalSetXLogInsertAllowed(void);
static void CreateEndOfRecoveryRecord(void);
static void CheckPointGuts(XLogRecPtr checkPointRedo, int flags);
+static XLogSegNo GetOldestKeepSegment(XLogRecPtr currpos, XLogRecPtr minSlotPtr);
static void KeepLogSeg(XLogRecPtr recptr, XLogSegNo *logSegNo);
static XLogRecPtr XLogGetReplicationSlotMinimumLSN(void);
@@ -9424,6 +9426,51 @@ CreateRestartPoint(int flags)
return true;
}
+/*
+ * Returns minimum segment number the next checktpoint must leave considering
+ * wal_keep_segments, replication slots and max_slot_wal_keep_size.
+ */
+static XLogSegNo
+GetOldestKeepSegment(XLogRecPtr currLSN, XLogRecPtr minSlotLSN)
+{
+ uint64 keepSegs = 0;
+ XLogSegNo currSeg;
+ XLogSegNo slotSeg;
+
+ XLByteToSeg(currLSN, currSeg, wal_segment_size);
+ XLByteToSeg(minSlotLSN, slotSeg, wal_segment_size);
+
+ /*
+ * Calcualte keep segments by slots first. The second term of the
+ * condition is just a sanity check.
+ */
+ if (minSlotLSN != InvalidXLogRecPtr && slotSeg <= currSeg)
+ keepSegs = currSeg - slotSeg;
+
+ /*
+ * slot keep segments is limited by max_slot_wal_keep_size, fragment of a
+ * segment is ignored
+ */
+ if (max_slot_wal_keep_size_mb > 0)
+ {
+ uint64 limitSegs;
+
+ limitSegs = ConvertToXSegs(max_slot_wal_keep_size_mb, wal_segment_size);
+ if (limitSegs < keepSegs)
+ keepSegs = limitSegs;
+ }
+
+ /* but, keep larger than wal_segment_size if any*/
+ if (wal_keep_segments > 0 && keepSegs < wal_keep_segments)
+ keepSegs = wal_keep_segments;
+
+ /* avoid underflow, don't go below 1 */
+ if (currSeg <= keepSegs)
+ return 1;
+
+ return currSeg - keepSegs;
+}
+
/*
* Retreat *logSegNo to the last segment that we need to retain because of
* either wal_keep_segments or replication slots.
@@ -9436,33 +9483,46 @@ static void
KeepLogSeg(XLogRecPtr recptr, XLogSegNo *logSegNo)
{
XLogSegNo segno;
- XLogRecPtr keep;
+ XLogRecPtr slotminptr = InvalidXLogRecPtr;
+ XLogSegNo minSegNo;
+ XLogSegNo slotSegNo;
XLByteToSeg(recptr, segno, wal_segment_size);
- keep = XLogGetReplicationSlotMinimumLSN();
- /* compute limit for wal_keep_segments first */
- if (wal_keep_segments > 0)
+ if (max_replication_slots > 0)
+ slotminptr = XLogGetReplicationSlotMinimumLSN();
+
+ /*
+ * We should keep certain number of WAL segments after this checktpoint.
+ */
+ minSegNo = GetOldestKeepSegment(recptr, slotminptr);
+
+ /*
+ * warn if the checkpoint flushes the segments required by replication
+ * slots.
+ */
+ if (!XLogRecPtrIsInvalid(slotminptr))
{
- /* avoid underflow, don't go below 1 */
- if (segno <= wal_keep_segments)
- segno = 1;
+ static XLogSegNo prev_lost_segs = 0; /* avoid duplicate messages */
+
+ XLByteToSeg(slotminptr, slotSegNo, wal_segment_size);
+
+ if (slotSegNo < minSegNo)
+ {
+ XLogSegNo lost_segs = minSegNo - slotSegNo;
+ if (prev_lost_segs != lost_segs)
+ ereport(WARNING,
+ (errmsg ("some replication slots have lost required WAL segments"),
+ errdetail("The mostly affected slot has lost %ld segments.",
+ lost_segs)));
+ prev_lost_segs = lost_segs;
+ }
else
- segno = segno - wal_keep_segments;
+ prev_lost_segs = 0;
}
- /* then check whether slots limit removal further */
- if (max_replication_slots > 0 && keep != InvalidXLogRecPtr)
- {
- XLogSegNo slotSegNo;
-
- XLByteToSeg(keep, slotSegNo, wal_segment_size);
-
- if (slotSegNo <= 0)
- segno = 1;
- else if (slotSegNo < segno)
- segno = slotSegNo;
- }
+ if (minSegNo < segno)
+ segno = minSegNo;
/* don't delete WAL segments newer than the calculated segment */
if (segno < *logSegNo)
diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c
index 17292e04fe..01b8c8edec 100644
--- a/src/backend/utils/misc/guc.c
+++ b/src/backend/utils/misc/guc.c
@@ -2530,6 +2530,18 @@ static struct config_int ConfigureNamesInt[] =
NULL, NULL, NULL
},
+ {
+ {"max_slot_wal_keep_size", PGC_SIGHUP, REPLICATION_SENDING,
+ gettext_noop("Sets the maximum size of extra WALs kept by replication slots."),
+ NULL,
+ GUC_UNIT_MB
+ },
+ &max_slot_wal_keep_size_mb,
+ 0, 0,
+ MAX_KILOBYTES, /* XXX: This is in megabytes, like max/min_wal_size */
+ NULL, NULL, NULL
+ },
+
{
{"wal_sender_timeout", PGC_SIGHUP, REPLICATION_SENDING,
gettext_noop("Sets the maximum time to wait for WAL replication."),
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index 9e39baf466..0e605a1765 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -239,6 +239,7 @@
#max_wal_senders = 10 # max number of walsender processes
# (change requires restart)
#wal_keep_segments = 0 # in logfile segments; 0 disables
+#max_slot_wal_keep_size = 0 # measured in bytes; 0 disables
#wal_sender_timeout = 60s # in milliseconds; 0 disables
#max_replication_slots = 10 # max number of replication slots
diff --git a/src/include/access/xlog.h b/src/include/access/xlog.h
index 421ba6d775..12cd0d1d10 100644
--- a/src/include/access/xlog.h
+++ b/src/include/access/xlog.h
@@ -98,6 +98,7 @@ extern int wal_segment_size;
extern int min_wal_size_mb;
extern int max_wal_size_mb;
extern int wal_keep_segments;
+extern int max_slot_wal_keep_size_mb;
extern int XLOGbuffers;
extern int XLogArchiveTimeout;
extern int wal_retrieve_retry_interval;
--
2.16.3
----Next_Part(Mon_Jul_09_14_47_06_2018_788)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
filename="v4-0002-Add-monitoring-aid-for-max_replication_slots.patch"
view thread (9+ messages) latest in thread
Message-ID: <no-message-id-753603@localhost>
Permalink: ../no-message-id-753603@localhost/
Also on: postgresql.org/message-id/no-message-id-753603@localhost
reply
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Reply to all the recipients using the --to and --cc options:
reply via email
To: pgsql-hackers@postgresql.org
Cc: horiguchi.kyotaro@lab.ntt.co.jp
Subject: Re: [PATCH 1/4] Add WAL releaf vent for replication slots
In-Reply-To: <no-message-id-753603@localhost>
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
This inbox is served by agora; see mirroring instructions
for how to clone and mirror all data and code used for this inbox