($INBOX_DIR/description missing)
help / color / mirror / Atom feed[PATCH v1] Replace min_safe_lsn with "distance" in pg_replication_slots
32+ messages / 6 participants
[nested] [flat]
* [PATCH v1] Replace min_safe_lsn with "distance" in pg_replication_slots
@ 2020-06-30 12:09 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 32+ messages in thread
From: Kyotaro Horiguchi @ 2020-06-30 12:09 UTC (permalink / raw)
pg_replication_slot.min_safe_lsn, which shows the oldest LSN kept in
pg_wal, is doubtful in usability for monitoring. Change it to
distance, which shows how many bytes the server can advance before the
slot loses required segments.
---
src/backend/access/transam/xlog.c | 39 +++++++++++++++++++++++
src/backend/catalog/system_views.sql | 2 +-
src/backend/replication/slotfuncs.c | 19 +++++------
src/include/access/xlog.h | 1 +
src/include/catalog/pg_proc.dat | 4 +--
src/test/recovery/t/019_replslot_limit.pl | 20 ++++++------
src/test/regress/expected/rules.out | 4 +--
7 files changed, 62 insertions(+), 27 deletions(-)
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index fd93bcfaeb..1f27639912 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -9570,6 +9570,45 @@ GetWALAvailability(XLogRecPtr targetLSN)
return WALAVAIL_REMOVED;
}
+/*
+ * Calculate how many bytes we can advance from currptr until the targetLSN is
+ * removed.
+ *
+ * Returns 0 if the distance is invalid.
+ */
+uint64
+DistanceToWALHorizon(XLogRecPtr targetLSN, XLogRecPtr currptr)
+{
+ XLogSegNo targetSeg;
+ XLogSegNo keepSegs;
+ XLogSegNo failSeg;
+ XLogRecPtr horizon;
+
+ XLByteToSeg(targetLSN, targetSeg, wal_segment_size);
+ keepSegs = 0;
+
+ /* no limit if max_slot_wal_keep_size is invalid */
+ if (max_slot_wal_keep_size_mb < 0)
+ return 0;
+
+ /* How many segments slots can keep? */
+ keepSegs = ConvertToXSegs(max_slot_wal_keep_size_mb, wal_segment_size);
+
+ /* override by wal_keep_segments if needed */
+ if (wal_keep_segments > keepSegs)
+ keepSegs = wal_keep_segments;
+
+ /* calculate the LSN where targetLSN is lost when currpos reaches */
+ failSeg = targetSeg + keepSegs + 1;
+ XLogSegNoOffsetToRecPtr(failSeg, 0, wal_segment_size, horizon);
+
+ /* If currptr already beyond the horizon, return zero. */
+ if (currptr > horizon)
+ return 0;
+
+ /* return the distance from currptr to the horizon */
+ return horizon - currptr;
+}
/*
* Retreat *logSegNo to the last segment that we need to retain because of
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index 5314e9348f..b9847a9f92 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -879,7 +879,7 @@ CREATE VIEW pg_replication_slots AS
L.restart_lsn,
L.confirmed_flush_lsn,
L.wal_status,
- L.min_safe_lsn
+ L.distance
FROM pg_get_replication_slots() AS L
LEFT JOIN pg_database D ON (L.datoid = D.oid);
diff --git a/src/backend/replication/slotfuncs.c b/src/backend/replication/slotfuncs.c
index 88033a79b2..532b3c5826 100644
--- a/src/backend/replication/slotfuncs.c
+++ b/src/backend/replication/slotfuncs.c
@@ -242,6 +242,7 @@ pg_get_replication_slots(PG_FUNCTION_ARGS)
Tuplestorestate *tupstore;
MemoryContext per_query_ctx;
MemoryContext oldcontext;
+ XLogRecPtr currlsn;
int slotno;
/* check to see if caller supports us returning a tuplestore */
@@ -274,6 +275,8 @@ pg_get_replication_slots(PG_FUNCTION_ARGS)
MemoryContextSwitchTo(oldcontext);
+ currlsn = GetXLogWriteRecPtr();
+
LWLockAcquire(ReplicationSlotControlLock, LW_SHARED);
for (slotno = 0; slotno < max_replication_slots; slotno++)
{
@@ -282,7 +285,7 @@ pg_get_replication_slots(PG_FUNCTION_ARGS)
Datum values[PG_GET_REPLICATION_SLOTS_COLS];
bool nulls[PG_GET_REPLICATION_SLOTS_COLS];
WALAvailability walstate;
- XLogSegNo last_removed_seg;
+ uint32 distance;
int i;
if (!slot->in_use)
@@ -398,16 +401,10 @@ pg_get_replication_slots(PG_FUNCTION_ARGS)
break;
}
- if (max_slot_wal_keep_size_mb >= 0 &&
- (walstate == WALAVAIL_RESERVED || walstate == WALAVAIL_EXTENDED) &&
- ((last_removed_seg = XLogGetLastRemovedSegno()) != 0))
- {
- XLogRecPtr min_safe_lsn;
-
- XLogSegNoOffsetToRecPtr(last_removed_seg + 1, 0,
- wal_segment_size, min_safe_lsn);
- values[i++] = Int64GetDatum(min_safe_lsn);
- }
+ distance =
+ DistanceToWALHorizon(slot_contents.data.restart_lsn, currlsn);
+ if (distance > 0)
+ values[i++] = Int64GetDatum(distance);
else
nulls[i++] = true;
diff --git a/src/include/access/xlog.h b/src/include/access/xlog.h
index 77ac4e785f..1ec448c5d5 100644
--- a/src/include/access/xlog.h
+++ b/src/include/access/xlog.h
@@ -329,6 +329,7 @@ extern void InitXLOGAccess(void);
extern void CreateCheckPoint(int flags);
extern bool CreateRestartPoint(int flags);
extern WALAvailability GetWALAvailability(XLogRecPtr targetLSN);
+extern uint64 DistanceToWALHorizon(XLogRecPtr targetLSN, XLogRecPtr currptr);
extern XLogRecPtr CalculateMaxmumSafeLSN(void);
extern void XLogPutNextOid(Oid nextOid);
extern XLogRecPtr XLogRestorePoint(const char *rpName);
diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat
index 61f2c2f5b4..199fd994bd 100644
--- a/src/include/catalog/pg_proc.dat
+++ b/src/include/catalog/pg_proc.dat
@@ -10063,9 +10063,9 @@
proname => 'pg_get_replication_slots', prorows => '10', proisstrict => 'f',
proretset => 't', provolatile => 's', prorettype => 'record',
proargtypes => '',
- proallargtypes => '{name,name,text,oid,bool,bool,int4,xid,xid,pg_lsn,pg_lsn,text,pg_lsn}',
+ proallargtypes => '{name,name,text,oid,bool,bool,int4,xid,xid,pg_lsn,pg_lsn,text,int8}',
proargmodes => '{o,o,o,o,o,o,o,o,o,o,o,o,o}',
- proargnames => '{slot_name,plugin,slot_type,datoid,temporary,active,active_pid,xmin,catalog_xmin,restart_lsn,confirmed_flush_lsn,wal_status,min_safe_lsn}',
+ proargnames => '{slot_name,plugin,slot_type,datoid,temporary,active,active_pid,xmin,catalog_xmin,restart_lsn,confirmed_flush_lsn,wal_status,distance}',
prosrc => 'pg_get_replication_slots' },
{ oid => '3786', descr => 'set up a logical replication slot',
proname => 'pg_create_logical_replication_slot', provolatile => 'v',
diff --git a/src/test/recovery/t/019_replslot_limit.pl b/src/test/recovery/t/019_replslot_limit.pl
index 7d22ae5720..1c76d2d9e9 100644
--- a/src/test/recovery/t/019_replslot_limit.pl
+++ b/src/test/recovery/t/019_replslot_limit.pl
@@ -28,7 +28,7 @@ $node_master->safe_psql('postgres',
# The slot state and remain should be null before the first connection
my $result = $node_master->safe_psql('postgres',
- "SELECT restart_lsn IS NULL, wal_status is NULL, min_safe_lsn is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
+ "SELECT restart_lsn IS NULL, wal_status is NULL, distance is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
);
is($result, "t|t|t", 'check the state of non-reserved slot is "unknown"');
@@ -52,9 +52,9 @@ $node_master->wait_for_catchup($node_standby, 'replay', $start_lsn);
# Stop standby
$node_standby->stop;
-# Preparation done, the slot is the state "normal" now
+# Preparation done, the slot is the state "reserved" now
$result = $node_master->safe_psql('postgres',
- "SELECT wal_status, min_safe_lsn is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
+ "SELECT wal_status, distance is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
);
is($result, "reserved|t", 'check the catching-up state');
@@ -64,7 +64,7 @@ $node_master->safe_psql('postgres', "CHECKPOINT;");
# The slot is always "safe" when fitting max_wal_size
$result = $node_master->safe_psql('postgres',
- "SELECT wal_status, min_safe_lsn is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
+ "SELECT wal_status, distance is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
);
is($result, "reserved|t",
'check that it is safe if WAL fits in max_wal_size');
@@ -74,7 +74,7 @@ $node_master->safe_psql('postgres', "CHECKPOINT;");
# The slot is always "safe" when max_slot_wal_keep_size is not set
$result = $node_master->safe_psql('postgres',
- "SELECT wal_status, min_safe_lsn is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
+ "SELECT wal_status, distance is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
);
is($result, "reserved|t", 'check that slot is working');
@@ -94,9 +94,7 @@ max_slot_wal_keep_size = ${max_slot_wal_keep_size_mb}MB
));
$node_master->reload;
-# The slot is in safe state. The distance from the min_safe_lsn should
-# be as almost (max_slot_wal_keep_size - 1) times large as the segment
-# size
+# The slot is in safe state.
$result = $node_master->safe_psql('postgres',
"SELECT wal_status FROM pg_replication_slots WHERE slot_name = 'rep1'");
@@ -110,7 +108,7 @@ $node_master->safe_psql('postgres', "CHECKPOINT;");
$result = $node_master->safe_psql('postgres',
"SELECT wal_status FROM pg_replication_slots WHERE slot_name = 'rep1'");
is($result, "reserved",
- 'check that min_safe_lsn gets close to the current LSN');
+ 'check that distance gets close to the current LSN');
# The standby can reconnect to master
$node_standby->start;
@@ -154,7 +152,7 @@ advance_wal($node_master, 1);
# Slot gets into 'unreserved' state
$result = $node_master->safe_psql('postgres',
- "SELECT wal_status, min_safe_lsn is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
+ "SELECT wal_status, distance is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
);
is($result, "unreserved|t",
'check that the slot state changes to "unreserved"');
@@ -186,7 +184,7 @@ ok( find_in_log(
# This slot should be broken
$result = $node_master->safe_psql('postgres',
- "SELECT slot_name, active, restart_lsn IS NULL, wal_status, min_safe_lsn FROM pg_replication_slots WHERE slot_name = 'rep1'"
+ "SELECT slot_name, active, restart_lsn IS NULL, wal_status, distance FROM pg_replication_slots WHERE slot_name = 'rep1'"
);
is($result, "rep1|f|t|lost|",
'check that the slot became inactive and the state "lost" persists');
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b813e32215..392eab12dd 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1464,8 +1464,8 @@ pg_replication_slots| SELECT l.slot_name,
l.restart_lsn,
l.confirmed_flush_lsn,
l.wal_status,
- l.min_safe_lsn
- FROM (pg_get_replication_slots() l(slot_name, plugin, slot_type, datoid, temporary, active, active_pid, xmin, catalog_xmin, restart_lsn, confirmed_flush_lsn, wal_status, min_safe_lsn)
+ l.distance
+ FROM (pg_get_replication_slots() l(slot_name, plugin, slot_type, datoid, temporary, active, active_pid, xmin, catalog_xmin, restart_lsn, confirmed_flush_lsn, wal_status, distance)
LEFT JOIN pg_database d ON ((l.datoid = d.oid)));
pg_roles| SELECT pg_authid.rolname,
pg_authid.rolsuper,
--
2.18.4
----Next_Part(Wed_Jul__1_10_32_59_2020_161)----
^ permalink raw reply [nested|flat] 32+ messages in thread
* [PATCH v1] Replace min_safe_lsn with "distance" in pg_replication_slots
@ 2020-06-30 12:09 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 32+ messages in thread
From: Kyotaro Horiguchi @ 2020-06-30 12:09 UTC (permalink / raw)
pg_replication_slot.min_safe_lsn, which shows the oldest LSN kept in
pg_wal, is doubtful in usability for monitoring. Change it to
distance, which shows how many bytes the server can advance before the
slot loses required segments.
---
src/backend/access/transam/xlog.c | 39 +++++++++++++++++++++++
src/backend/catalog/system_views.sql | 2 +-
src/backend/replication/slotfuncs.c | 19 +++++------
src/include/access/xlog.h | 1 +
src/include/catalog/pg_proc.dat | 4 +--
src/test/recovery/t/019_replslot_limit.pl | 20 ++++++------
src/test/regress/expected/rules.out | 4 +--
7 files changed, 62 insertions(+), 27 deletions(-)
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index fd93bcfaeb..1f27639912 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -9570,6 +9570,45 @@ GetWALAvailability(XLogRecPtr targetLSN)
return WALAVAIL_REMOVED;
}
+/*
+ * Calculate how many bytes we can advance from currptr until the targetLSN is
+ * removed.
+ *
+ * Returns 0 if the distance is invalid.
+ */
+uint64
+DistanceToWALHorizon(XLogRecPtr targetLSN, XLogRecPtr currptr)
+{
+ XLogSegNo targetSeg;
+ XLogSegNo keepSegs;
+ XLogSegNo failSeg;
+ XLogRecPtr horizon;
+
+ XLByteToSeg(targetLSN, targetSeg, wal_segment_size);
+ keepSegs = 0;
+
+ /* no limit if max_slot_wal_keep_size is invalid */
+ if (max_slot_wal_keep_size_mb < 0)
+ return 0;
+
+ /* How many segments slots can keep? */
+ keepSegs = ConvertToXSegs(max_slot_wal_keep_size_mb, wal_segment_size);
+
+ /* override by wal_keep_segments if needed */
+ if (wal_keep_segments > keepSegs)
+ keepSegs = wal_keep_segments;
+
+ /* calculate the LSN where targetLSN is lost when currpos reaches */
+ failSeg = targetSeg + keepSegs + 1;
+ XLogSegNoOffsetToRecPtr(failSeg, 0, wal_segment_size, horizon);
+
+ /* If currptr already beyond the horizon, return zero. */
+ if (currptr > horizon)
+ return 0;
+
+ /* return the distance from currptr to the horizon */
+ return horizon - currptr;
+}
/*
* Retreat *logSegNo to the last segment that we need to retain because of
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index 5314e9348f..b9847a9f92 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -879,7 +879,7 @@ CREATE VIEW pg_replication_slots AS
L.restart_lsn,
L.confirmed_flush_lsn,
L.wal_status,
- L.min_safe_lsn
+ L.distance
FROM pg_get_replication_slots() AS L
LEFT JOIN pg_database D ON (L.datoid = D.oid);
diff --git a/src/backend/replication/slotfuncs.c b/src/backend/replication/slotfuncs.c
index 88033a79b2..532b3c5826 100644
--- a/src/backend/replication/slotfuncs.c
+++ b/src/backend/replication/slotfuncs.c
@@ -242,6 +242,7 @@ pg_get_replication_slots(PG_FUNCTION_ARGS)
Tuplestorestate *tupstore;
MemoryContext per_query_ctx;
MemoryContext oldcontext;
+ XLogRecPtr currlsn;
int slotno;
/* check to see if caller supports us returning a tuplestore */
@@ -274,6 +275,8 @@ pg_get_replication_slots(PG_FUNCTION_ARGS)
MemoryContextSwitchTo(oldcontext);
+ currlsn = GetXLogWriteRecPtr();
+
LWLockAcquire(ReplicationSlotControlLock, LW_SHARED);
for (slotno = 0; slotno < max_replication_slots; slotno++)
{
@@ -282,7 +285,7 @@ pg_get_replication_slots(PG_FUNCTION_ARGS)
Datum values[PG_GET_REPLICATION_SLOTS_COLS];
bool nulls[PG_GET_REPLICATION_SLOTS_COLS];
WALAvailability walstate;
- XLogSegNo last_removed_seg;
+ uint32 distance;
int i;
if (!slot->in_use)
@@ -398,16 +401,10 @@ pg_get_replication_slots(PG_FUNCTION_ARGS)
break;
}
- if (max_slot_wal_keep_size_mb >= 0 &&
- (walstate == WALAVAIL_RESERVED || walstate == WALAVAIL_EXTENDED) &&
- ((last_removed_seg = XLogGetLastRemovedSegno()) != 0))
- {
- XLogRecPtr min_safe_lsn;
-
- XLogSegNoOffsetToRecPtr(last_removed_seg + 1, 0,
- wal_segment_size, min_safe_lsn);
- values[i++] = Int64GetDatum(min_safe_lsn);
- }
+ distance =
+ DistanceToWALHorizon(slot_contents.data.restart_lsn, currlsn);
+ if (distance > 0)
+ values[i++] = Int64GetDatum(distance);
else
nulls[i++] = true;
diff --git a/src/include/access/xlog.h b/src/include/access/xlog.h
index 77ac4e785f..1ec448c5d5 100644
--- a/src/include/access/xlog.h
+++ b/src/include/access/xlog.h
@@ -329,6 +329,7 @@ extern void InitXLOGAccess(void);
extern void CreateCheckPoint(int flags);
extern bool CreateRestartPoint(int flags);
extern WALAvailability GetWALAvailability(XLogRecPtr targetLSN);
+extern uint64 DistanceToWALHorizon(XLogRecPtr targetLSN, XLogRecPtr currptr);
extern XLogRecPtr CalculateMaxmumSafeLSN(void);
extern void XLogPutNextOid(Oid nextOid);
extern XLogRecPtr XLogRestorePoint(const char *rpName);
diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat
index 61f2c2f5b4..199fd994bd 100644
--- a/src/include/catalog/pg_proc.dat
+++ b/src/include/catalog/pg_proc.dat
@@ -10063,9 +10063,9 @@
proname => 'pg_get_replication_slots', prorows => '10', proisstrict => 'f',
proretset => 't', provolatile => 's', prorettype => 'record',
proargtypes => '',
- proallargtypes => '{name,name,text,oid,bool,bool,int4,xid,xid,pg_lsn,pg_lsn,text,pg_lsn}',
+ proallargtypes => '{name,name,text,oid,bool,bool,int4,xid,xid,pg_lsn,pg_lsn,text,int8}',
proargmodes => '{o,o,o,o,o,o,o,o,o,o,o,o,o}',
- proargnames => '{slot_name,plugin,slot_type,datoid,temporary,active,active_pid,xmin,catalog_xmin,restart_lsn,confirmed_flush_lsn,wal_status,min_safe_lsn}',
+ proargnames => '{slot_name,plugin,slot_type,datoid,temporary,active,active_pid,xmin,catalog_xmin,restart_lsn,confirmed_flush_lsn,wal_status,distance}',
prosrc => 'pg_get_replication_slots' },
{ oid => '3786', descr => 'set up a logical replication slot',
proname => 'pg_create_logical_replication_slot', provolatile => 'v',
diff --git a/src/test/recovery/t/019_replslot_limit.pl b/src/test/recovery/t/019_replslot_limit.pl
index 7d22ae5720..1c76d2d9e9 100644
--- a/src/test/recovery/t/019_replslot_limit.pl
+++ b/src/test/recovery/t/019_replslot_limit.pl
@@ -28,7 +28,7 @@ $node_master->safe_psql('postgres',
# The slot state and remain should be null before the first connection
my $result = $node_master->safe_psql('postgres',
- "SELECT restart_lsn IS NULL, wal_status is NULL, min_safe_lsn is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
+ "SELECT restart_lsn IS NULL, wal_status is NULL, distance is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
);
is($result, "t|t|t", 'check the state of non-reserved slot is "unknown"');
@@ -52,9 +52,9 @@ $node_master->wait_for_catchup($node_standby, 'replay', $start_lsn);
# Stop standby
$node_standby->stop;
-# Preparation done, the slot is the state "normal" now
+# Preparation done, the slot is the state "reserved" now
$result = $node_master->safe_psql('postgres',
- "SELECT wal_status, min_safe_lsn is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
+ "SELECT wal_status, distance is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
);
is($result, "reserved|t", 'check the catching-up state');
@@ -64,7 +64,7 @@ $node_master->safe_psql('postgres', "CHECKPOINT;");
# The slot is always "safe" when fitting max_wal_size
$result = $node_master->safe_psql('postgres',
- "SELECT wal_status, min_safe_lsn is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
+ "SELECT wal_status, distance is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
);
is($result, "reserved|t",
'check that it is safe if WAL fits in max_wal_size');
@@ -74,7 +74,7 @@ $node_master->safe_psql('postgres', "CHECKPOINT;");
# The slot is always "safe" when max_slot_wal_keep_size is not set
$result = $node_master->safe_psql('postgres',
- "SELECT wal_status, min_safe_lsn is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
+ "SELECT wal_status, distance is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
);
is($result, "reserved|t", 'check that slot is working');
@@ -94,9 +94,7 @@ max_slot_wal_keep_size = ${max_slot_wal_keep_size_mb}MB
));
$node_master->reload;
-# The slot is in safe state. The distance from the min_safe_lsn should
-# be as almost (max_slot_wal_keep_size - 1) times large as the segment
-# size
+# The slot is in safe state.
$result = $node_master->safe_psql('postgres',
"SELECT wal_status FROM pg_replication_slots WHERE slot_name = 'rep1'");
@@ -110,7 +108,7 @@ $node_master->safe_psql('postgres', "CHECKPOINT;");
$result = $node_master->safe_psql('postgres',
"SELECT wal_status FROM pg_replication_slots WHERE slot_name = 'rep1'");
is($result, "reserved",
- 'check that min_safe_lsn gets close to the current LSN');
+ 'check that distance gets close to the current LSN');
# The standby can reconnect to master
$node_standby->start;
@@ -154,7 +152,7 @@ advance_wal($node_master, 1);
# Slot gets into 'unreserved' state
$result = $node_master->safe_psql('postgres',
- "SELECT wal_status, min_safe_lsn is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
+ "SELECT wal_status, distance is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
);
is($result, "unreserved|t",
'check that the slot state changes to "unreserved"');
@@ -186,7 +184,7 @@ ok( find_in_log(
# This slot should be broken
$result = $node_master->safe_psql('postgres',
- "SELECT slot_name, active, restart_lsn IS NULL, wal_status, min_safe_lsn FROM pg_replication_slots WHERE slot_name = 'rep1'"
+ "SELECT slot_name, active, restart_lsn IS NULL, wal_status, distance FROM pg_replication_slots WHERE slot_name = 'rep1'"
);
is($result, "rep1|f|t|lost|",
'check that the slot became inactive and the state "lost" persists');
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b813e32215..392eab12dd 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1464,8 +1464,8 @@ pg_replication_slots| SELECT l.slot_name,
l.restart_lsn,
l.confirmed_flush_lsn,
l.wal_status,
- l.min_safe_lsn
- FROM (pg_get_replication_slots() l(slot_name, plugin, slot_type, datoid, temporary, active, active_pid, xmin, catalog_xmin, restart_lsn, confirmed_flush_lsn, wal_status, min_safe_lsn)
+ l.distance
+ FROM (pg_get_replication_slots() l(slot_name, plugin, slot_type, datoid, temporary, active, active_pid, xmin, catalog_xmin, restart_lsn, confirmed_flush_lsn, wal_status, distance)
LEFT JOIN pg_database d ON ((l.datoid = d.oid)));
pg_roles| SELECT pg_authid.rolname,
pg_authid.rolsuper,
--
2.18.4
----Next_Part(Wed_Jul__1_10_32_59_2020_161)----
^ permalink raw reply [nested|flat] 32+ messages in thread
* [PATCH v1] Replace min_safe_lsn with "distance" in pg_replication_slots
@ 2020-06-30 12:09 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 32+ messages in thread
From: Kyotaro Horiguchi @ 2020-06-30 12:09 UTC (permalink / raw)
pg_replication_slot.min_safe_lsn, which shows the oldest LSN kept in
pg_wal, is doubtful in usability for monitoring. Change it to
distance, which shows how many bytes the server can advance before the
slot loses required segments.
---
src/backend/access/transam/xlog.c | 39 +++++++++++++++++++++++
src/backend/catalog/system_views.sql | 2 +-
src/backend/replication/slotfuncs.c | 19 +++++------
src/include/access/xlog.h | 1 +
src/include/catalog/pg_proc.dat | 4 +--
src/test/recovery/t/019_replslot_limit.pl | 20 ++++++------
src/test/regress/expected/rules.out | 4 +--
7 files changed, 62 insertions(+), 27 deletions(-)
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index fd93bcfaeb..1f27639912 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -9570,6 +9570,45 @@ GetWALAvailability(XLogRecPtr targetLSN)
return WALAVAIL_REMOVED;
}
+/*
+ * Calculate how many bytes we can advance from currptr until the targetLSN is
+ * removed.
+ *
+ * Returns 0 if the distance is invalid.
+ */
+uint64
+DistanceToWALHorizon(XLogRecPtr targetLSN, XLogRecPtr currptr)
+{
+ XLogSegNo targetSeg;
+ XLogSegNo keepSegs;
+ XLogSegNo failSeg;
+ XLogRecPtr horizon;
+
+ XLByteToSeg(targetLSN, targetSeg, wal_segment_size);
+ keepSegs = 0;
+
+ /* no limit if max_slot_wal_keep_size is invalid */
+ if (max_slot_wal_keep_size_mb < 0)
+ return 0;
+
+ /* How many segments slots can keep? */
+ keepSegs = ConvertToXSegs(max_slot_wal_keep_size_mb, wal_segment_size);
+
+ /* override by wal_keep_segments if needed */
+ if (wal_keep_segments > keepSegs)
+ keepSegs = wal_keep_segments;
+
+ /* calculate the LSN where targetLSN is lost when currpos reaches */
+ failSeg = targetSeg + keepSegs + 1;
+ XLogSegNoOffsetToRecPtr(failSeg, 0, wal_segment_size, horizon);
+
+ /* If currptr already beyond the horizon, return zero. */
+ if (currptr > horizon)
+ return 0;
+
+ /* return the distance from currptr to the horizon */
+ return horizon - currptr;
+}
/*
* Retreat *logSegNo to the last segment that we need to retain because of
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index 5314e9348f..b9847a9f92 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -879,7 +879,7 @@ CREATE VIEW pg_replication_slots AS
L.restart_lsn,
L.confirmed_flush_lsn,
L.wal_status,
- L.min_safe_lsn
+ L.distance
FROM pg_get_replication_slots() AS L
LEFT JOIN pg_database D ON (L.datoid = D.oid);
diff --git a/src/backend/replication/slotfuncs.c b/src/backend/replication/slotfuncs.c
index 88033a79b2..532b3c5826 100644
--- a/src/backend/replication/slotfuncs.c
+++ b/src/backend/replication/slotfuncs.c
@@ -242,6 +242,7 @@ pg_get_replication_slots(PG_FUNCTION_ARGS)
Tuplestorestate *tupstore;
MemoryContext per_query_ctx;
MemoryContext oldcontext;
+ XLogRecPtr currlsn;
int slotno;
/* check to see if caller supports us returning a tuplestore */
@@ -274,6 +275,8 @@ pg_get_replication_slots(PG_FUNCTION_ARGS)
MemoryContextSwitchTo(oldcontext);
+ currlsn = GetXLogWriteRecPtr();
+
LWLockAcquire(ReplicationSlotControlLock, LW_SHARED);
for (slotno = 0; slotno < max_replication_slots; slotno++)
{
@@ -282,7 +285,7 @@ pg_get_replication_slots(PG_FUNCTION_ARGS)
Datum values[PG_GET_REPLICATION_SLOTS_COLS];
bool nulls[PG_GET_REPLICATION_SLOTS_COLS];
WALAvailability walstate;
- XLogSegNo last_removed_seg;
+ uint32 distance;
int i;
if (!slot->in_use)
@@ -398,16 +401,10 @@ pg_get_replication_slots(PG_FUNCTION_ARGS)
break;
}
- if (max_slot_wal_keep_size_mb >= 0 &&
- (walstate == WALAVAIL_RESERVED || walstate == WALAVAIL_EXTENDED) &&
- ((last_removed_seg = XLogGetLastRemovedSegno()) != 0))
- {
- XLogRecPtr min_safe_lsn;
-
- XLogSegNoOffsetToRecPtr(last_removed_seg + 1, 0,
- wal_segment_size, min_safe_lsn);
- values[i++] = Int64GetDatum(min_safe_lsn);
- }
+ distance =
+ DistanceToWALHorizon(slot_contents.data.restart_lsn, currlsn);
+ if (distance > 0)
+ values[i++] = Int64GetDatum(distance);
else
nulls[i++] = true;
diff --git a/src/include/access/xlog.h b/src/include/access/xlog.h
index 77ac4e785f..1ec448c5d5 100644
--- a/src/include/access/xlog.h
+++ b/src/include/access/xlog.h
@@ -329,6 +329,7 @@ extern void InitXLOGAccess(void);
extern void CreateCheckPoint(int flags);
extern bool CreateRestartPoint(int flags);
extern WALAvailability GetWALAvailability(XLogRecPtr targetLSN);
+extern uint64 DistanceToWALHorizon(XLogRecPtr targetLSN, XLogRecPtr currptr);
extern XLogRecPtr CalculateMaxmumSafeLSN(void);
extern void XLogPutNextOid(Oid nextOid);
extern XLogRecPtr XLogRestorePoint(const char *rpName);
diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat
index 61f2c2f5b4..199fd994bd 100644
--- a/src/include/catalog/pg_proc.dat
+++ b/src/include/catalog/pg_proc.dat
@@ -10063,9 +10063,9 @@
proname => 'pg_get_replication_slots', prorows => '10', proisstrict => 'f',
proretset => 't', provolatile => 's', prorettype => 'record',
proargtypes => '',
- proallargtypes => '{name,name,text,oid,bool,bool,int4,xid,xid,pg_lsn,pg_lsn,text,pg_lsn}',
+ proallargtypes => '{name,name,text,oid,bool,bool,int4,xid,xid,pg_lsn,pg_lsn,text,int8}',
proargmodes => '{o,o,o,o,o,o,o,o,o,o,o,o,o}',
- proargnames => '{slot_name,plugin,slot_type,datoid,temporary,active,active_pid,xmin,catalog_xmin,restart_lsn,confirmed_flush_lsn,wal_status,min_safe_lsn}',
+ proargnames => '{slot_name,plugin,slot_type,datoid,temporary,active,active_pid,xmin,catalog_xmin,restart_lsn,confirmed_flush_lsn,wal_status,distance}',
prosrc => 'pg_get_replication_slots' },
{ oid => '3786', descr => 'set up a logical replication slot',
proname => 'pg_create_logical_replication_slot', provolatile => 'v',
diff --git a/src/test/recovery/t/019_replslot_limit.pl b/src/test/recovery/t/019_replslot_limit.pl
index 7d22ae5720..1c76d2d9e9 100644
--- a/src/test/recovery/t/019_replslot_limit.pl
+++ b/src/test/recovery/t/019_replslot_limit.pl
@@ -28,7 +28,7 @@ $node_master->safe_psql('postgres',
# The slot state and remain should be null before the first connection
my $result = $node_master->safe_psql('postgres',
- "SELECT restart_lsn IS NULL, wal_status is NULL, min_safe_lsn is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
+ "SELECT restart_lsn IS NULL, wal_status is NULL, distance is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
);
is($result, "t|t|t", 'check the state of non-reserved slot is "unknown"');
@@ -52,9 +52,9 @@ $node_master->wait_for_catchup($node_standby, 'replay', $start_lsn);
# Stop standby
$node_standby->stop;
-# Preparation done, the slot is the state "normal" now
+# Preparation done, the slot is the state "reserved" now
$result = $node_master->safe_psql('postgres',
- "SELECT wal_status, min_safe_lsn is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
+ "SELECT wal_status, distance is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
);
is($result, "reserved|t", 'check the catching-up state');
@@ -64,7 +64,7 @@ $node_master->safe_psql('postgres', "CHECKPOINT;");
# The slot is always "safe" when fitting max_wal_size
$result = $node_master->safe_psql('postgres',
- "SELECT wal_status, min_safe_lsn is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
+ "SELECT wal_status, distance is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
);
is($result, "reserved|t",
'check that it is safe if WAL fits in max_wal_size');
@@ -74,7 +74,7 @@ $node_master->safe_psql('postgres', "CHECKPOINT;");
# The slot is always "safe" when max_slot_wal_keep_size is not set
$result = $node_master->safe_psql('postgres',
- "SELECT wal_status, min_safe_lsn is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
+ "SELECT wal_status, distance is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
);
is($result, "reserved|t", 'check that slot is working');
@@ -94,9 +94,7 @@ max_slot_wal_keep_size = ${max_slot_wal_keep_size_mb}MB
));
$node_master->reload;
-# The slot is in safe state. The distance from the min_safe_lsn should
-# be as almost (max_slot_wal_keep_size - 1) times large as the segment
-# size
+# The slot is in safe state.
$result = $node_master->safe_psql('postgres',
"SELECT wal_status FROM pg_replication_slots WHERE slot_name = 'rep1'");
@@ -110,7 +108,7 @@ $node_master->safe_psql('postgres', "CHECKPOINT;");
$result = $node_master->safe_psql('postgres',
"SELECT wal_status FROM pg_replication_slots WHERE slot_name = 'rep1'");
is($result, "reserved",
- 'check that min_safe_lsn gets close to the current LSN');
+ 'check that distance gets close to the current LSN');
# The standby can reconnect to master
$node_standby->start;
@@ -154,7 +152,7 @@ advance_wal($node_master, 1);
# Slot gets into 'unreserved' state
$result = $node_master->safe_psql('postgres',
- "SELECT wal_status, min_safe_lsn is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
+ "SELECT wal_status, distance is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
);
is($result, "unreserved|t",
'check that the slot state changes to "unreserved"');
@@ -186,7 +184,7 @@ ok( find_in_log(
# This slot should be broken
$result = $node_master->safe_psql('postgres',
- "SELECT slot_name, active, restart_lsn IS NULL, wal_status, min_safe_lsn FROM pg_replication_slots WHERE slot_name = 'rep1'"
+ "SELECT slot_name, active, restart_lsn IS NULL, wal_status, distance FROM pg_replication_slots WHERE slot_name = 'rep1'"
);
is($result, "rep1|f|t|lost|",
'check that the slot became inactive and the state "lost" persists');
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b813e32215..392eab12dd 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1464,8 +1464,8 @@ pg_replication_slots| SELECT l.slot_name,
l.restart_lsn,
l.confirmed_flush_lsn,
l.wal_status,
- l.min_safe_lsn
- FROM (pg_get_replication_slots() l(slot_name, plugin, slot_type, datoid, temporary, active, active_pid, xmin, catalog_xmin, restart_lsn, confirmed_flush_lsn, wal_status, min_safe_lsn)
+ l.distance
+ FROM (pg_get_replication_slots() l(slot_name, plugin, slot_type, datoid, temporary, active, active_pid, xmin, catalog_xmin, restart_lsn, confirmed_flush_lsn, wal_status, distance)
LEFT JOIN pg_database d ON ((l.datoid = d.oid)));
pg_roles| SELECT pg_authid.rolname,
pg_authid.rolsuper,
--
2.18.4
----Next_Part(Wed_Jul__1_10_32_59_2020_161)----
^ permalink raw reply [nested|flat] 32+ messages in thread
* [PATCH v1] Replace min_safe_lsn with "distance" in pg_replication_slots
@ 2020-06-30 12:09 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 32+ messages in thread
From: Kyotaro Horiguchi @ 2020-06-30 12:09 UTC (permalink / raw)
pg_replication_slot.min_safe_lsn, which shows the oldest LSN kept in
pg_wal, is doubtful in usability for monitoring. Change it to
distance, which shows how many bytes the server can advance before the
slot loses required segments.
---
src/backend/access/transam/xlog.c | 39 +++++++++++++++++++++++
src/backend/catalog/system_views.sql | 2 +-
src/backend/replication/slotfuncs.c | 19 +++++------
src/include/access/xlog.h | 1 +
src/include/catalog/pg_proc.dat | 4 +--
src/test/recovery/t/019_replslot_limit.pl | 20 ++++++------
src/test/regress/expected/rules.out | 4 +--
7 files changed, 62 insertions(+), 27 deletions(-)
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index fd93bcfaeb..1f27639912 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -9570,6 +9570,45 @@ GetWALAvailability(XLogRecPtr targetLSN)
return WALAVAIL_REMOVED;
}
+/*
+ * Calculate how many bytes we can advance from currptr until the targetLSN is
+ * removed.
+ *
+ * Returns 0 if the distance is invalid.
+ */
+uint64
+DistanceToWALHorizon(XLogRecPtr targetLSN, XLogRecPtr currptr)
+{
+ XLogSegNo targetSeg;
+ XLogSegNo keepSegs;
+ XLogSegNo failSeg;
+ XLogRecPtr horizon;
+
+ XLByteToSeg(targetLSN, targetSeg, wal_segment_size);
+ keepSegs = 0;
+
+ /* no limit if max_slot_wal_keep_size is invalid */
+ if (max_slot_wal_keep_size_mb < 0)
+ return 0;
+
+ /* How many segments slots can keep? */
+ keepSegs = ConvertToXSegs(max_slot_wal_keep_size_mb, wal_segment_size);
+
+ /* override by wal_keep_segments if needed */
+ if (wal_keep_segments > keepSegs)
+ keepSegs = wal_keep_segments;
+
+ /* calculate the LSN where targetLSN is lost when currpos reaches */
+ failSeg = targetSeg + keepSegs + 1;
+ XLogSegNoOffsetToRecPtr(failSeg, 0, wal_segment_size, horizon);
+
+ /* If currptr already beyond the horizon, return zero. */
+ if (currptr > horizon)
+ return 0;
+
+ /* return the distance from currptr to the horizon */
+ return horizon - currptr;
+}
/*
* Retreat *logSegNo to the last segment that we need to retain because of
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index 5314e9348f..b9847a9f92 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -879,7 +879,7 @@ CREATE VIEW pg_replication_slots AS
L.restart_lsn,
L.confirmed_flush_lsn,
L.wal_status,
- L.min_safe_lsn
+ L.distance
FROM pg_get_replication_slots() AS L
LEFT JOIN pg_database D ON (L.datoid = D.oid);
diff --git a/src/backend/replication/slotfuncs.c b/src/backend/replication/slotfuncs.c
index 88033a79b2..532b3c5826 100644
--- a/src/backend/replication/slotfuncs.c
+++ b/src/backend/replication/slotfuncs.c
@@ -242,6 +242,7 @@ pg_get_replication_slots(PG_FUNCTION_ARGS)
Tuplestorestate *tupstore;
MemoryContext per_query_ctx;
MemoryContext oldcontext;
+ XLogRecPtr currlsn;
int slotno;
/* check to see if caller supports us returning a tuplestore */
@@ -274,6 +275,8 @@ pg_get_replication_slots(PG_FUNCTION_ARGS)
MemoryContextSwitchTo(oldcontext);
+ currlsn = GetXLogWriteRecPtr();
+
LWLockAcquire(ReplicationSlotControlLock, LW_SHARED);
for (slotno = 0; slotno < max_replication_slots; slotno++)
{
@@ -282,7 +285,7 @@ pg_get_replication_slots(PG_FUNCTION_ARGS)
Datum values[PG_GET_REPLICATION_SLOTS_COLS];
bool nulls[PG_GET_REPLICATION_SLOTS_COLS];
WALAvailability walstate;
- XLogSegNo last_removed_seg;
+ uint32 distance;
int i;
if (!slot->in_use)
@@ -398,16 +401,10 @@ pg_get_replication_slots(PG_FUNCTION_ARGS)
break;
}
- if (max_slot_wal_keep_size_mb >= 0 &&
- (walstate == WALAVAIL_RESERVED || walstate == WALAVAIL_EXTENDED) &&
- ((last_removed_seg = XLogGetLastRemovedSegno()) != 0))
- {
- XLogRecPtr min_safe_lsn;
-
- XLogSegNoOffsetToRecPtr(last_removed_seg + 1, 0,
- wal_segment_size, min_safe_lsn);
- values[i++] = Int64GetDatum(min_safe_lsn);
- }
+ distance =
+ DistanceToWALHorizon(slot_contents.data.restart_lsn, currlsn);
+ if (distance > 0)
+ values[i++] = Int64GetDatum(distance);
else
nulls[i++] = true;
diff --git a/src/include/access/xlog.h b/src/include/access/xlog.h
index 77ac4e785f..1ec448c5d5 100644
--- a/src/include/access/xlog.h
+++ b/src/include/access/xlog.h
@@ -329,6 +329,7 @@ extern void InitXLOGAccess(void);
extern void CreateCheckPoint(int flags);
extern bool CreateRestartPoint(int flags);
extern WALAvailability GetWALAvailability(XLogRecPtr targetLSN);
+extern uint64 DistanceToWALHorizon(XLogRecPtr targetLSN, XLogRecPtr currptr);
extern XLogRecPtr CalculateMaxmumSafeLSN(void);
extern void XLogPutNextOid(Oid nextOid);
extern XLogRecPtr XLogRestorePoint(const char *rpName);
diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat
index 61f2c2f5b4..199fd994bd 100644
--- a/src/include/catalog/pg_proc.dat
+++ b/src/include/catalog/pg_proc.dat
@@ -10063,9 +10063,9 @@
proname => 'pg_get_replication_slots', prorows => '10', proisstrict => 'f',
proretset => 't', provolatile => 's', prorettype => 'record',
proargtypes => '',
- proallargtypes => '{name,name,text,oid,bool,bool,int4,xid,xid,pg_lsn,pg_lsn,text,pg_lsn}',
+ proallargtypes => '{name,name,text,oid,bool,bool,int4,xid,xid,pg_lsn,pg_lsn,text,int8}',
proargmodes => '{o,o,o,o,o,o,o,o,o,o,o,o,o}',
- proargnames => '{slot_name,plugin,slot_type,datoid,temporary,active,active_pid,xmin,catalog_xmin,restart_lsn,confirmed_flush_lsn,wal_status,min_safe_lsn}',
+ proargnames => '{slot_name,plugin,slot_type,datoid,temporary,active,active_pid,xmin,catalog_xmin,restart_lsn,confirmed_flush_lsn,wal_status,distance}',
prosrc => 'pg_get_replication_slots' },
{ oid => '3786', descr => 'set up a logical replication slot',
proname => 'pg_create_logical_replication_slot', provolatile => 'v',
diff --git a/src/test/recovery/t/019_replslot_limit.pl b/src/test/recovery/t/019_replslot_limit.pl
index 7d22ae5720..1c76d2d9e9 100644
--- a/src/test/recovery/t/019_replslot_limit.pl
+++ b/src/test/recovery/t/019_replslot_limit.pl
@@ -28,7 +28,7 @@ $node_master->safe_psql('postgres',
# The slot state and remain should be null before the first connection
my $result = $node_master->safe_psql('postgres',
- "SELECT restart_lsn IS NULL, wal_status is NULL, min_safe_lsn is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
+ "SELECT restart_lsn IS NULL, wal_status is NULL, distance is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
);
is($result, "t|t|t", 'check the state of non-reserved slot is "unknown"');
@@ -52,9 +52,9 @@ $node_master->wait_for_catchup($node_standby, 'replay', $start_lsn);
# Stop standby
$node_standby->stop;
-# Preparation done, the slot is the state "normal" now
+# Preparation done, the slot is the state "reserved" now
$result = $node_master->safe_psql('postgres',
- "SELECT wal_status, min_safe_lsn is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
+ "SELECT wal_status, distance is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
);
is($result, "reserved|t", 'check the catching-up state');
@@ -64,7 +64,7 @@ $node_master->safe_psql('postgres', "CHECKPOINT;");
# The slot is always "safe" when fitting max_wal_size
$result = $node_master->safe_psql('postgres',
- "SELECT wal_status, min_safe_lsn is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
+ "SELECT wal_status, distance is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
);
is($result, "reserved|t",
'check that it is safe if WAL fits in max_wal_size');
@@ -74,7 +74,7 @@ $node_master->safe_psql('postgres', "CHECKPOINT;");
# The slot is always "safe" when max_slot_wal_keep_size is not set
$result = $node_master->safe_psql('postgres',
- "SELECT wal_status, min_safe_lsn is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
+ "SELECT wal_status, distance is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
);
is($result, "reserved|t", 'check that slot is working');
@@ -94,9 +94,7 @@ max_slot_wal_keep_size = ${max_slot_wal_keep_size_mb}MB
));
$node_master->reload;
-# The slot is in safe state. The distance from the min_safe_lsn should
-# be as almost (max_slot_wal_keep_size - 1) times large as the segment
-# size
+# The slot is in safe state.
$result = $node_master->safe_psql('postgres',
"SELECT wal_status FROM pg_replication_slots WHERE slot_name = 'rep1'");
@@ -110,7 +108,7 @@ $node_master->safe_psql('postgres', "CHECKPOINT;");
$result = $node_master->safe_psql('postgres',
"SELECT wal_status FROM pg_replication_slots WHERE slot_name = 'rep1'");
is($result, "reserved",
- 'check that min_safe_lsn gets close to the current LSN');
+ 'check that distance gets close to the current LSN');
# The standby can reconnect to master
$node_standby->start;
@@ -154,7 +152,7 @@ advance_wal($node_master, 1);
# Slot gets into 'unreserved' state
$result = $node_master->safe_psql('postgres',
- "SELECT wal_status, min_safe_lsn is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
+ "SELECT wal_status, distance is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
);
is($result, "unreserved|t",
'check that the slot state changes to "unreserved"');
@@ -186,7 +184,7 @@ ok( find_in_log(
# This slot should be broken
$result = $node_master->safe_psql('postgres',
- "SELECT slot_name, active, restart_lsn IS NULL, wal_status, min_safe_lsn FROM pg_replication_slots WHERE slot_name = 'rep1'"
+ "SELECT slot_name, active, restart_lsn IS NULL, wal_status, distance FROM pg_replication_slots WHERE slot_name = 'rep1'"
);
is($result, "rep1|f|t|lost|",
'check that the slot became inactive and the state "lost" persists');
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b813e32215..392eab12dd 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1464,8 +1464,8 @@ pg_replication_slots| SELECT l.slot_name,
l.restart_lsn,
l.confirmed_flush_lsn,
l.wal_status,
- l.min_safe_lsn
- FROM (pg_get_replication_slots() l(slot_name, plugin, slot_type, datoid, temporary, active, active_pid, xmin, catalog_xmin, restart_lsn, confirmed_flush_lsn, wal_status, min_safe_lsn)
+ l.distance
+ FROM (pg_get_replication_slots() l(slot_name, plugin, slot_type, datoid, temporary, active, active_pid, xmin, catalog_xmin, restart_lsn, confirmed_flush_lsn, wal_status, distance)
LEFT JOIN pg_database d ON ((l.datoid = d.oid)));
pg_roles| SELECT pg_authid.rolname,
pg_authid.rolsuper,
--
2.18.4
----Next_Part(Wed_Jul__1_10_32_59_2020_161)----
^ permalink raw reply [nested|flat] 32+ messages in thread
* [PATCH v1] Replace min_safe_lsn with "distance" in pg_replication_slots
@ 2020-06-30 12:09 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 32+ messages in thread
From: Kyotaro Horiguchi @ 2020-06-30 12:09 UTC (permalink / raw)
pg_replication_slot.min_safe_lsn, which shows the oldest LSN kept in
pg_wal, is doubtful in usability for monitoring. Change it to
distance, which shows how many bytes the server can advance before the
slot loses required segments.
---
src/backend/access/transam/xlog.c | 39 +++++++++++++++++++++++
src/backend/catalog/system_views.sql | 2 +-
src/backend/replication/slotfuncs.c | 19 +++++------
src/include/access/xlog.h | 1 +
src/include/catalog/pg_proc.dat | 4 +--
src/test/recovery/t/019_replslot_limit.pl | 20 ++++++------
src/test/regress/expected/rules.out | 4 +--
7 files changed, 62 insertions(+), 27 deletions(-)
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index fd93bcfaeb..1f27639912 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -9570,6 +9570,45 @@ GetWALAvailability(XLogRecPtr targetLSN)
return WALAVAIL_REMOVED;
}
+/*
+ * Calculate how many bytes we can advance from currptr until the targetLSN is
+ * removed.
+ *
+ * Returns 0 if the distance is invalid.
+ */
+uint64
+DistanceToWALHorizon(XLogRecPtr targetLSN, XLogRecPtr currptr)
+{
+ XLogSegNo targetSeg;
+ XLogSegNo keepSegs;
+ XLogSegNo failSeg;
+ XLogRecPtr horizon;
+
+ XLByteToSeg(targetLSN, targetSeg, wal_segment_size);
+ keepSegs = 0;
+
+ /* no limit if max_slot_wal_keep_size is invalid */
+ if (max_slot_wal_keep_size_mb < 0)
+ return 0;
+
+ /* How many segments slots can keep? */
+ keepSegs = ConvertToXSegs(max_slot_wal_keep_size_mb, wal_segment_size);
+
+ /* override by wal_keep_segments if needed */
+ if (wal_keep_segments > keepSegs)
+ keepSegs = wal_keep_segments;
+
+ /* calculate the LSN where targetLSN is lost when currpos reaches */
+ failSeg = targetSeg + keepSegs + 1;
+ XLogSegNoOffsetToRecPtr(failSeg, 0, wal_segment_size, horizon);
+
+ /* If currptr already beyond the horizon, return zero. */
+ if (currptr > horizon)
+ return 0;
+
+ /* return the distance from currptr to the horizon */
+ return horizon - currptr;
+}
/*
* Retreat *logSegNo to the last segment that we need to retain because of
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index 5314e9348f..b9847a9f92 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -879,7 +879,7 @@ CREATE VIEW pg_replication_slots AS
L.restart_lsn,
L.confirmed_flush_lsn,
L.wal_status,
- L.min_safe_lsn
+ L.distance
FROM pg_get_replication_slots() AS L
LEFT JOIN pg_database D ON (L.datoid = D.oid);
diff --git a/src/backend/replication/slotfuncs.c b/src/backend/replication/slotfuncs.c
index 88033a79b2..532b3c5826 100644
--- a/src/backend/replication/slotfuncs.c
+++ b/src/backend/replication/slotfuncs.c
@@ -242,6 +242,7 @@ pg_get_replication_slots(PG_FUNCTION_ARGS)
Tuplestorestate *tupstore;
MemoryContext per_query_ctx;
MemoryContext oldcontext;
+ XLogRecPtr currlsn;
int slotno;
/* check to see if caller supports us returning a tuplestore */
@@ -274,6 +275,8 @@ pg_get_replication_slots(PG_FUNCTION_ARGS)
MemoryContextSwitchTo(oldcontext);
+ currlsn = GetXLogWriteRecPtr();
+
LWLockAcquire(ReplicationSlotControlLock, LW_SHARED);
for (slotno = 0; slotno < max_replication_slots; slotno++)
{
@@ -282,7 +285,7 @@ pg_get_replication_slots(PG_FUNCTION_ARGS)
Datum values[PG_GET_REPLICATION_SLOTS_COLS];
bool nulls[PG_GET_REPLICATION_SLOTS_COLS];
WALAvailability walstate;
- XLogSegNo last_removed_seg;
+ uint32 distance;
int i;
if (!slot->in_use)
@@ -398,16 +401,10 @@ pg_get_replication_slots(PG_FUNCTION_ARGS)
break;
}
- if (max_slot_wal_keep_size_mb >= 0 &&
- (walstate == WALAVAIL_RESERVED || walstate == WALAVAIL_EXTENDED) &&
- ((last_removed_seg = XLogGetLastRemovedSegno()) != 0))
- {
- XLogRecPtr min_safe_lsn;
-
- XLogSegNoOffsetToRecPtr(last_removed_seg + 1, 0,
- wal_segment_size, min_safe_lsn);
- values[i++] = Int64GetDatum(min_safe_lsn);
- }
+ distance =
+ DistanceToWALHorizon(slot_contents.data.restart_lsn, currlsn);
+ if (distance > 0)
+ values[i++] = Int64GetDatum(distance);
else
nulls[i++] = true;
diff --git a/src/include/access/xlog.h b/src/include/access/xlog.h
index 77ac4e785f..1ec448c5d5 100644
--- a/src/include/access/xlog.h
+++ b/src/include/access/xlog.h
@@ -329,6 +329,7 @@ extern void InitXLOGAccess(void);
extern void CreateCheckPoint(int flags);
extern bool CreateRestartPoint(int flags);
extern WALAvailability GetWALAvailability(XLogRecPtr targetLSN);
+extern uint64 DistanceToWALHorizon(XLogRecPtr targetLSN, XLogRecPtr currptr);
extern XLogRecPtr CalculateMaxmumSafeLSN(void);
extern void XLogPutNextOid(Oid nextOid);
extern XLogRecPtr XLogRestorePoint(const char *rpName);
diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat
index 61f2c2f5b4..199fd994bd 100644
--- a/src/include/catalog/pg_proc.dat
+++ b/src/include/catalog/pg_proc.dat
@@ -10063,9 +10063,9 @@
proname => 'pg_get_replication_slots', prorows => '10', proisstrict => 'f',
proretset => 't', provolatile => 's', prorettype => 'record',
proargtypes => '',
- proallargtypes => '{name,name,text,oid,bool,bool,int4,xid,xid,pg_lsn,pg_lsn,text,pg_lsn}',
+ proallargtypes => '{name,name,text,oid,bool,bool,int4,xid,xid,pg_lsn,pg_lsn,text,int8}',
proargmodes => '{o,o,o,o,o,o,o,o,o,o,o,o,o}',
- proargnames => '{slot_name,plugin,slot_type,datoid,temporary,active,active_pid,xmin,catalog_xmin,restart_lsn,confirmed_flush_lsn,wal_status,min_safe_lsn}',
+ proargnames => '{slot_name,plugin,slot_type,datoid,temporary,active,active_pid,xmin,catalog_xmin,restart_lsn,confirmed_flush_lsn,wal_status,distance}',
prosrc => 'pg_get_replication_slots' },
{ oid => '3786', descr => 'set up a logical replication slot',
proname => 'pg_create_logical_replication_slot', provolatile => 'v',
diff --git a/src/test/recovery/t/019_replslot_limit.pl b/src/test/recovery/t/019_replslot_limit.pl
index 7d22ae5720..1c76d2d9e9 100644
--- a/src/test/recovery/t/019_replslot_limit.pl
+++ b/src/test/recovery/t/019_replslot_limit.pl
@@ -28,7 +28,7 @@ $node_master->safe_psql('postgres',
# The slot state and remain should be null before the first connection
my $result = $node_master->safe_psql('postgres',
- "SELECT restart_lsn IS NULL, wal_status is NULL, min_safe_lsn is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
+ "SELECT restart_lsn IS NULL, wal_status is NULL, distance is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
);
is($result, "t|t|t", 'check the state of non-reserved slot is "unknown"');
@@ -52,9 +52,9 @@ $node_master->wait_for_catchup($node_standby, 'replay', $start_lsn);
# Stop standby
$node_standby->stop;
-# Preparation done, the slot is the state "normal" now
+# Preparation done, the slot is the state "reserved" now
$result = $node_master->safe_psql('postgres',
- "SELECT wal_status, min_safe_lsn is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
+ "SELECT wal_status, distance is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
);
is($result, "reserved|t", 'check the catching-up state');
@@ -64,7 +64,7 @@ $node_master->safe_psql('postgres', "CHECKPOINT;");
# The slot is always "safe" when fitting max_wal_size
$result = $node_master->safe_psql('postgres',
- "SELECT wal_status, min_safe_lsn is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
+ "SELECT wal_status, distance is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
);
is($result, "reserved|t",
'check that it is safe if WAL fits in max_wal_size');
@@ -74,7 +74,7 @@ $node_master->safe_psql('postgres', "CHECKPOINT;");
# The slot is always "safe" when max_slot_wal_keep_size is not set
$result = $node_master->safe_psql('postgres',
- "SELECT wal_status, min_safe_lsn is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
+ "SELECT wal_status, distance is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
);
is($result, "reserved|t", 'check that slot is working');
@@ -94,9 +94,7 @@ max_slot_wal_keep_size = ${max_slot_wal_keep_size_mb}MB
));
$node_master->reload;
-# The slot is in safe state. The distance from the min_safe_lsn should
-# be as almost (max_slot_wal_keep_size - 1) times large as the segment
-# size
+# The slot is in safe state.
$result = $node_master->safe_psql('postgres',
"SELECT wal_status FROM pg_replication_slots WHERE slot_name = 'rep1'");
@@ -110,7 +108,7 @@ $node_master->safe_psql('postgres', "CHECKPOINT;");
$result = $node_master->safe_psql('postgres',
"SELECT wal_status FROM pg_replication_slots WHERE slot_name = 'rep1'");
is($result, "reserved",
- 'check that min_safe_lsn gets close to the current LSN');
+ 'check that distance gets close to the current LSN');
# The standby can reconnect to master
$node_standby->start;
@@ -154,7 +152,7 @@ advance_wal($node_master, 1);
# Slot gets into 'unreserved' state
$result = $node_master->safe_psql('postgres',
- "SELECT wal_status, min_safe_lsn is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
+ "SELECT wal_status, distance is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
);
is($result, "unreserved|t",
'check that the slot state changes to "unreserved"');
@@ -186,7 +184,7 @@ ok( find_in_log(
# This slot should be broken
$result = $node_master->safe_psql('postgres',
- "SELECT slot_name, active, restart_lsn IS NULL, wal_status, min_safe_lsn FROM pg_replication_slots WHERE slot_name = 'rep1'"
+ "SELECT slot_name, active, restart_lsn IS NULL, wal_status, distance FROM pg_replication_slots WHERE slot_name = 'rep1'"
);
is($result, "rep1|f|t|lost|",
'check that the slot became inactive and the state "lost" persists');
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b813e32215..392eab12dd 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1464,8 +1464,8 @@ pg_replication_slots| SELECT l.slot_name,
l.restart_lsn,
l.confirmed_flush_lsn,
l.wal_status,
- l.min_safe_lsn
- FROM (pg_get_replication_slots() l(slot_name, plugin, slot_type, datoid, temporary, active, active_pid, xmin, catalog_xmin, restart_lsn, confirmed_flush_lsn, wal_status, min_safe_lsn)
+ l.distance
+ FROM (pg_get_replication_slots() l(slot_name, plugin, slot_type, datoid, temporary, active, active_pid, xmin, catalog_xmin, restart_lsn, confirmed_flush_lsn, wal_status, distance)
LEFT JOIN pg_database d ON ((l.datoid = d.oid)));
pg_roles| SELECT pg_authid.rolname,
pg_authid.rolsuper,
--
2.18.4
----Next_Part(Wed_Jul__1_10_32_59_2020_161)----
^ permalink raw reply [nested|flat] 32+ messages in thread
* [PATCH v1] Replace min_safe_lsn with "distance" in pg_replication_slots
@ 2020-06-30 12:09 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 32+ messages in thread
From: Kyotaro Horiguchi @ 2020-06-30 12:09 UTC (permalink / raw)
pg_replication_slot.min_safe_lsn, which shows the oldest LSN kept in
pg_wal, is doubtful in usability for monitoring. Change it to
distance, which shows how many bytes the server can advance before the
slot loses required segments.
---
src/backend/access/transam/xlog.c | 39 +++++++++++++++++++++++
src/backend/catalog/system_views.sql | 2 +-
src/backend/replication/slotfuncs.c | 19 +++++------
src/include/access/xlog.h | 1 +
src/include/catalog/pg_proc.dat | 4 +--
src/test/recovery/t/019_replslot_limit.pl | 20 ++++++------
src/test/regress/expected/rules.out | 4 +--
7 files changed, 62 insertions(+), 27 deletions(-)
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index fd93bcfaeb..1f27639912 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -9570,6 +9570,45 @@ GetWALAvailability(XLogRecPtr targetLSN)
return WALAVAIL_REMOVED;
}
+/*
+ * Calculate how many bytes we can advance from currptr until the targetLSN is
+ * removed.
+ *
+ * Returns 0 if the distance is invalid.
+ */
+uint64
+DistanceToWALHorizon(XLogRecPtr targetLSN, XLogRecPtr currptr)
+{
+ XLogSegNo targetSeg;
+ XLogSegNo keepSegs;
+ XLogSegNo failSeg;
+ XLogRecPtr horizon;
+
+ XLByteToSeg(targetLSN, targetSeg, wal_segment_size);
+ keepSegs = 0;
+
+ /* no limit if max_slot_wal_keep_size is invalid */
+ if (max_slot_wal_keep_size_mb < 0)
+ return 0;
+
+ /* How many segments slots can keep? */
+ keepSegs = ConvertToXSegs(max_slot_wal_keep_size_mb, wal_segment_size);
+
+ /* override by wal_keep_segments if needed */
+ if (wal_keep_segments > keepSegs)
+ keepSegs = wal_keep_segments;
+
+ /* calculate the LSN where targetLSN is lost when currpos reaches */
+ failSeg = targetSeg + keepSegs + 1;
+ XLogSegNoOffsetToRecPtr(failSeg, 0, wal_segment_size, horizon);
+
+ /* If currptr already beyond the horizon, return zero. */
+ if (currptr > horizon)
+ return 0;
+
+ /* return the distance from currptr to the horizon */
+ return horizon - currptr;
+}
/*
* Retreat *logSegNo to the last segment that we need to retain because of
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index 5314e9348f..b9847a9f92 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -879,7 +879,7 @@ CREATE VIEW pg_replication_slots AS
L.restart_lsn,
L.confirmed_flush_lsn,
L.wal_status,
- L.min_safe_lsn
+ L.distance
FROM pg_get_replication_slots() AS L
LEFT JOIN pg_database D ON (L.datoid = D.oid);
diff --git a/src/backend/replication/slotfuncs.c b/src/backend/replication/slotfuncs.c
index 88033a79b2..532b3c5826 100644
--- a/src/backend/replication/slotfuncs.c
+++ b/src/backend/replication/slotfuncs.c
@@ -242,6 +242,7 @@ pg_get_replication_slots(PG_FUNCTION_ARGS)
Tuplestorestate *tupstore;
MemoryContext per_query_ctx;
MemoryContext oldcontext;
+ XLogRecPtr currlsn;
int slotno;
/* check to see if caller supports us returning a tuplestore */
@@ -274,6 +275,8 @@ pg_get_replication_slots(PG_FUNCTION_ARGS)
MemoryContextSwitchTo(oldcontext);
+ currlsn = GetXLogWriteRecPtr();
+
LWLockAcquire(ReplicationSlotControlLock, LW_SHARED);
for (slotno = 0; slotno < max_replication_slots; slotno++)
{
@@ -282,7 +285,7 @@ pg_get_replication_slots(PG_FUNCTION_ARGS)
Datum values[PG_GET_REPLICATION_SLOTS_COLS];
bool nulls[PG_GET_REPLICATION_SLOTS_COLS];
WALAvailability walstate;
- XLogSegNo last_removed_seg;
+ uint32 distance;
int i;
if (!slot->in_use)
@@ -398,16 +401,10 @@ pg_get_replication_slots(PG_FUNCTION_ARGS)
break;
}
- if (max_slot_wal_keep_size_mb >= 0 &&
- (walstate == WALAVAIL_RESERVED || walstate == WALAVAIL_EXTENDED) &&
- ((last_removed_seg = XLogGetLastRemovedSegno()) != 0))
- {
- XLogRecPtr min_safe_lsn;
-
- XLogSegNoOffsetToRecPtr(last_removed_seg + 1, 0,
- wal_segment_size, min_safe_lsn);
- values[i++] = Int64GetDatum(min_safe_lsn);
- }
+ distance =
+ DistanceToWALHorizon(slot_contents.data.restart_lsn, currlsn);
+ if (distance > 0)
+ values[i++] = Int64GetDatum(distance);
else
nulls[i++] = true;
diff --git a/src/include/access/xlog.h b/src/include/access/xlog.h
index 77ac4e785f..1ec448c5d5 100644
--- a/src/include/access/xlog.h
+++ b/src/include/access/xlog.h
@@ -329,6 +329,7 @@ extern void InitXLOGAccess(void);
extern void CreateCheckPoint(int flags);
extern bool CreateRestartPoint(int flags);
extern WALAvailability GetWALAvailability(XLogRecPtr targetLSN);
+extern uint64 DistanceToWALHorizon(XLogRecPtr targetLSN, XLogRecPtr currptr);
extern XLogRecPtr CalculateMaxmumSafeLSN(void);
extern void XLogPutNextOid(Oid nextOid);
extern XLogRecPtr XLogRestorePoint(const char *rpName);
diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat
index 61f2c2f5b4..199fd994bd 100644
--- a/src/include/catalog/pg_proc.dat
+++ b/src/include/catalog/pg_proc.dat
@@ -10063,9 +10063,9 @@
proname => 'pg_get_replication_slots', prorows => '10', proisstrict => 'f',
proretset => 't', provolatile => 's', prorettype => 'record',
proargtypes => '',
- proallargtypes => '{name,name,text,oid,bool,bool,int4,xid,xid,pg_lsn,pg_lsn,text,pg_lsn}',
+ proallargtypes => '{name,name,text,oid,bool,bool,int4,xid,xid,pg_lsn,pg_lsn,text,int8}',
proargmodes => '{o,o,o,o,o,o,o,o,o,o,o,o,o}',
- proargnames => '{slot_name,plugin,slot_type,datoid,temporary,active,active_pid,xmin,catalog_xmin,restart_lsn,confirmed_flush_lsn,wal_status,min_safe_lsn}',
+ proargnames => '{slot_name,plugin,slot_type,datoid,temporary,active,active_pid,xmin,catalog_xmin,restart_lsn,confirmed_flush_lsn,wal_status,distance}',
prosrc => 'pg_get_replication_slots' },
{ oid => '3786', descr => 'set up a logical replication slot',
proname => 'pg_create_logical_replication_slot', provolatile => 'v',
diff --git a/src/test/recovery/t/019_replslot_limit.pl b/src/test/recovery/t/019_replslot_limit.pl
index 7d22ae5720..1c76d2d9e9 100644
--- a/src/test/recovery/t/019_replslot_limit.pl
+++ b/src/test/recovery/t/019_replslot_limit.pl
@@ -28,7 +28,7 @@ $node_master->safe_psql('postgres',
# The slot state and remain should be null before the first connection
my $result = $node_master->safe_psql('postgres',
- "SELECT restart_lsn IS NULL, wal_status is NULL, min_safe_lsn is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
+ "SELECT restart_lsn IS NULL, wal_status is NULL, distance is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
);
is($result, "t|t|t", 'check the state of non-reserved slot is "unknown"');
@@ -52,9 +52,9 @@ $node_master->wait_for_catchup($node_standby, 'replay', $start_lsn);
# Stop standby
$node_standby->stop;
-# Preparation done, the slot is the state "normal" now
+# Preparation done, the slot is the state "reserved" now
$result = $node_master->safe_psql('postgres',
- "SELECT wal_status, min_safe_lsn is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
+ "SELECT wal_status, distance is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
);
is($result, "reserved|t", 'check the catching-up state');
@@ -64,7 +64,7 @@ $node_master->safe_psql('postgres', "CHECKPOINT;");
# The slot is always "safe" when fitting max_wal_size
$result = $node_master->safe_psql('postgres',
- "SELECT wal_status, min_safe_lsn is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
+ "SELECT wal_status, distance is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
);
is($result, "reserved|t",
'check that it is safe if WAL fits in max_wal_size');
@@ -74,7 +74,7 @@ $node_master->safe_psql('postgres', "CHECKPOINT;");
# The slot is always "safe" when max_slot_wal_keep_size is not set
$result = $node_master->safe_psql('postgres',
- "SELECT wal_status, min_safe_lsn is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
+ "SELECT wal_status, distance is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
);
is($result, "reserved|t", 'check that slot is working');
@@ -94,9 +94,7 @@ max_slot_wal_keep_size = ${max_slot_wal_keep_size_mb}MB
));
$node_master->reload;
-# The slot is in safe state. The distance from the min_safe_lsn should
-# be as almost (max_slot_wal_keep_size - 1) times large as the segment
-# size
+# The slot is in safe state.
$result = $node_master->safe_psql('postgres',
"SELECT wal_status FROM pg_replication_slots WHERE slot_name = 'rep1'");
@@ -110,7 +108,7 @@ $node_master->safe_psql('postgres', "CHECKPOINT;");
$result = $node_master->safe_psql('postgres',
"SELECT wal_status FROM pg_replication_slots WHERE slot_name = 'rep1'");
is($result, "reserved",
- 'check that min_safe_lsn gets close to the current LSN');
+ 'check that distance gets close to the current LSN');
# The standby can reconnect to master
$node_standby->start;
@@ -154,7 +152,7 @@ advance_wal($node_master, 1);
# Slot gets into 'unreserved' state
$result = $node_master->safe_psql('postgres',
- "SELECT wal_status, min_safe_lsn is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
+ "SELECT wal_status, distance is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
);
is($result, "unreserved|t",
'check that the slot state changes to "unreserved"');
@@ -186,7 +184,7 @@ ok( find_in_log(
# This slot should be broken
$result = $node_master->safe_psql('postgres',
- "SELECT slot_name, active, restart_lsn IS NULL, wal_status, min_safe_lsn FROM pg_replication_slots WHERE slot_name = 'rep1'"
+ "SELECT slot_name, active, restart_lsn IS NULL, wal_status, distance FROM pg_replication_slots WHERE slot_name = 'rep1'"
);
is($result, "rep1|f|t|lost|",
'check that the slot became inactive and the state "lost" persists');
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b813e32215..392eab12dd 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1464,8 +1464,8 @@ pg_replication_slots| SELECT l.slot_name,
l.restart_lsn,
l.confirmed_flush_lsn,
l.wal_status,
- l.min_safe_lsn
- FROM (pg_get_replication_slots() l(slot_name, plugin, slot_type, datoid, temporary, active, active_pid, xmin, catalog_xmin, restart_lsn, confirmed_flush_lsn, wal_status, min_safe_lsn)
+ l.distance
+ FROM (pg_get_replication_slots() l(slot_name, plugin, slot_type, datoid, temporary, active, active_pid, xmin, catalog_xmin, restart_lsn, confirmed_flush_lsn, wal_status, distance)
LEFT JOIN pg_database d ON ((l.datoid = d.oid)));
pg_roles| SELECT pg_authid.rolname,
pg_authid.rolsuper,
--
2.18.4
----Next_Part(Wed_Jul__1_10_32_59_2020_161)----
^ permalink raw reply [nested|flat] 32+ messages in thread
* [PATCH v1] Replace min_safe_lsn with "distance" in pg_replication_slots
@ 2020-06-30 12:09 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 32+ messages in thread
From: Kyotaro Horiguchi @ 2020-06-30 12:09 UTC (permalink / raw)
pg_replication_slot.min_safe_lsn, which shows the oldest LSN kept in
pg_wal, is doubtful in usability for monitoring. Change it to
distance, which shows how many bytes the server can advance before the
slot loses required segments.
---
src/backend/access/transam/xlog.c | 39 +++++++++++++++++++++++
src/backend/catalog/system_views.sql | 2 +-
src/backend/replication/slotfuncs.c | 19 +++++------
src/include/access/xlog.h | 1 +
src/include/catalog/pg_proc.dat | 4 +--
src/test/recovery/t/019_replslot_limit.pl | 20 ++++++------
src/test/regress/expected/rules.out | 4 +--
7 files changed, 62 insertions(+), 27 deletions(-)
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index fd93bcfaeb..1f27639912 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -9570,6 +9570,45 @@ GetWALAvailability(XLogRecPtr targetLSN)
return WALAVAIL_REMOVED;
}
+/*
+ * Calculate how many bytes we can advance from currptr until the targetLSN is
+ * removed.
+ *
+ * Returns 0 if the distance is invalid.
+ */
+uint64
+DistanceToWALHorizon(XLogRecPtr targetLSN, XLogRecPtr currptr)
+{
+ XLogSegNo targetSeg;
+ XLogSegNo keepSegs;
+ XLogSegNo failSeg;
+ XLogRecPtr horizon;
+
+ XLByteToSeg(targetLSN, targetSeg, wal_segment_size);
+ keepSegs = 0;
+
+ /* no limit if max_slot_wal_keep_size is invalid */
+ if (max_slot_wal_keep_size_mb < 0)
+ return 0;
+
+ /* How many segments slots can keep? */
+ keepSegs = ConvertToXSegs(max_slot_wal_keep_size_mb, wal_segment_size);
+
+ /* override by wal_keep_segments if needed */
+ if (wal_keep_segments > keepSegs)
+ keepSegs = wal_keep_segments;
+
+ /* calculate the LSN where targetLSN is lost when currpos reaches */
+ failSeg = targetSeg + keepSegs + 1;
+ XLogSegNoOffsetToRecPtr(failSeg, 0, wal_segment_size, horizon);
+
+ /* If currptr already beyond the horizon, return zero. */
+ if (currptr > horizon)
+ return 0;
+
+ /* return the distance from currptr to the horizon */
+ return horizon - currptr;
+}
/*
* Retreat *logSegNo to the last segment that we need to retain because of
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index 5314e9348f..b9847a9f92 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -879,7 +879,7 @@ CREATE VIEW pg_replication_slots AS
L.restart_lsn,
L.confirmed_flush_lsn,
L.wal_status,
- L.min_safe_lsn
+ L.distance
FROM pg_get_replication_slots() AS L
LEFT JOIN pg_database D ON (L.datoid = D.oid);
diff --git a/src/backend/replication/slotfuncs.c b/src/backend/replication/slotfuncs.c
index 88033a79b2..532b3c5826 100644
--- a/src/backend/replication/slotfuncs.c
+++ b/src/backend/replication/slotfuncs.c
@@ -242,6 +242,7 @@ pg_get_replication_slots(PG_FUNCTION_ARGS)
Tuplestorestate *tupstore;
MemoryContext per_query_ctx;
MemoryContext oldcontext;
+ XLogRecPtr currlsn;
int slotno;
/* check to see if caller supports us returning a tuplestore */
@@ -274,6 +275,8 @@ pg_get_replication_slots(PG_FUNCTION_ARGS)
MemoryContextSwitchTo(oldcontext);
+ currlsn = GetXLogWriteRecPtr();
+
LWLockAcquire(ReplicationSlotControlLock, LW_SHARED);
for (slotno = 0; slotno < max_replication_slots; slotno++)
{
@@ -282,7 +285,7 @@ pg_get_replication_slots(PG_FUNCTION_ARGS)
Datum values[PG_GET_REPLICATION_SLOTS_COLS];
bool nulls[PG_GET_REPLICATION_SLOTS_COLS];
WALAvailability walstate;
- XLogSegNo last_removed_seg;
+ uint32 distance;
int i;
if (!slot->in_use)
@@ -398,16 +401,10 @@ pg_get_replication_slots(PG_FUNCTION_ARGS)
break;
}
- if (max_slot_wal_keep_size_mb >= 0 &&
- (walstate == WALAVAIL_RESERVED || walstate == WALAVAIL_EXTENDED) &&
- ((last_removed_seg = XLogGetLastRemovedSegno()) != 0))
- {
- XLogRecPtr min_safe_lsn;
-
- XLogSegNoOffsetToRecPtr(last_removed_seg + 1, 0,
- wal_segment_size, min_safe_lsn);
- values[i++] = Int64GetDatum(min_safe_lsn);
- }
+ distance =
+ DistanceToWALHorizon(slot_contents.data.restart_lsn, currlsn);
+ if (distance > 0)
+ values[i++] = Int64GetDatum(distance);
else
nulls[i++] = true;
diff --git a/src/include/access/xlog.h b/src/include/access/xlog.h
index 77ac4e785f..1ec448c5d5 100644
--- a/src/include/access/xlog.h
+++ b/src/include/access/xlog.h
@@ -329,6 +329,7 @@ extern void InitXLOGAccess(void);
extern void CreateCheckPoint(int flags);
extern bool CreateRestartPoint(int flags);
extern WALAvailability GetWALAvailability(XLogRecPtr targetLSN);
+extern uint64 DistanceToWALHorizon(XLogRecPtr targetLSN, XLogRecPtr currptr);
extern XLogRecPtr CalculateMaxmumSafeLSN(void);
extern void XLogPutNextOid(Oid nextOid);
extern XLogRecPtr XLogRestorePoint(const char *rpName);
diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat
index 61f2c2f5b4..199fd994bd 100644
--- a/src/include/catalog/pg_proc.dat
+++ b/src/include/catalog/pg_proc.dat
@@ -10063,9 +10063,9 @@
proname => 'pg_get_replication_slots', prorows => '10', proisstrict => 'f',
proretset => 't', provolatile => 's', prorettype => 'record',
proargtypes => '',
- proallargtypes => '{name,name,text,oid,bool,bool,int4,xid,xid,pg_lsn,pg_lsn,text,pg_lsn}',
+ proallargtypes => '{name,name,text,oid,bool,bool,int4,xid,xid,pg_lsn,pg_lsn,text,int8}',
proargmodes => '{o,o,o,o,o,o,o,o,o,o,o,o,o}',
- proargnames => '{slot_name,plugin,slot_type,datoid,temporary,active,active_pid,xmin,catalog_xmin,restart_lsn,confirmed_flush_lsn,wal_status,min_safe_lsn}',
+ proargnames => '{slot_name,plugin,slot_type,datoid,temporary,active,active_pid,xmin,catalog_xmin,restart_lsn,confirmed_flush_lsn,wal_status,distance}',
prosrc => 'pg_get_replication_slots' },
{ oid => '3786', descr => 'set up a logical replication slot',
proname => 'pg_create_logical_replication_slot', provolatile => 'v',
diff --git a/src/test/recovery/t/019_replslot_limit.pl b/src/test/recovery/t/019_replslot_limit.pl
index 7d22ae5720..1c76d2d9e9 100644
--- a/src/test/recovery/t/019_replslot_limit.pl
+++ b/src/test/recovery/t/019_replslot_limit.pl
@@ -28,7 +28,7 @@ $node_master->safe_psql('postgres',
# The slot state and remain should be null before the first connection
my $result = $node_master->safe_psql('postgres',
- "SELECT restart_lsn IS NULL, wal_status is NULL, min_safe_lsn is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
+ "SELECT restart_lsn IS NULL, wal_status is NULL, distance is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
);
is($result, "t|t|t", 'check the state of non-reserved slot is "unknown"');
@@ -52,9 +52,9 @@ $node_master->wait_for_catchup($node_standby, 'replay', $start_lsn);
# Stop standby
$node_standby->stop;
-# Preparation done, the slot is the state "normal" now
+# Preparation done, the slot is the state "reserved" now
$result = $node_master->safe_psql('postgres',
- "SELECT wal_status, min_safe_lsn is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
+ "SELECT wal_status, distance is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
);
is($result, "reserved|t", 'check the catching-up state');
@@ -64,7 +64,7 @@ $node_master->safe_psql('postgres', "CHECKPOINT;");
# The slot is always "safe" when fitting max_wal_size
$result = $node_master->safe_psql('postgres',
- "SELECT wal_status, min_safe_lsn is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
+ "SELECT wal_status, distance is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
);
is($result, "reserved|t",
'check that it is safe if WAL fits in max_wal_size');
@@ -74,7 +74,7 @@ $node_master->safe_psql('postgres', "CHECKPOINT;");
# The slot is always "safe" when max_slot_wal_keep_size is not set
$result = $node_master->safe_psql('postgres',
- "SELECT wal_status, min_safe_lsn is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
+ "SELECT wal_status, distance is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
);
is($result, "reserved|t", 'check that slot is working');
@@ -94,9 +94,7 @@ max_slot_wal_keep_size = ${max_slot_wal_keep_size_mb}MB
));
$node_master->reload;
-# The slot is in safe state. The distance from the min_safe_lsn should
-# be as almost (max_slot_wal_keep_size - 1) times large as the segment
-# size
+# The slot is in safe state.
$result = $node_master->safe_psql('postgres',
"SELECT wal_status FROM pg_replication_slots WHERE slot_name = 'rep1'");
@@ -110,7 +108,7 @@ $node_master->safe_psql('postgres', "CHECKPOINT;");
$result = $node_master->safe_psql('postgres',
"SELECT wal_status FROM pg_replication_slots WHERE slot_name = 'rep1'");
is($result, "reserved",
- 'check that min_safe_lsn gets close to the current LSN');
+ 'check that distance gets close to the current LSN');
# The standby can reconnect to master
$node_standby->start;
@@ -154,7 +152,7 @@ advance_wal($node_master, 1);
# Slot gets into 'unreserved' state
$result = $node_master->safe_psql('postgres',
- "SELECT wal_status, min_safe_lsn is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
+ "SELECT wal_status, distance is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
);
is($result, "unreserved|t",
'check that the slot state changes to "unreserved"');
@@ -186,7 +184,7 @@ ok( find_in_log(
# This slot should be broken
$result = $node_master->safe_psql('postgres',
- "SELECT slot_name, active, restart_lsn IS NULL, wal_status, min_safe_lsn FROM pg_replication_slots WHERE slot_name = 'rep1'"
+ "SELECT slot_name, active, restart_lsn IS NULL, wal_status, distance FROM pg_replication_slots WHERE slot_name = 'rep1'"
);
is($result, "rep1|f|t|lost|",
'check that the slot became inactive and the state "lost" persists');
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b813e32215..392eab12dd 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1464,8 +1464,8 @@ pg_replication_slots| SELECT l.slot_name,
l.restart_lsn,
l.confirmed_flush_lsn,
l.wal_status,
- l.min_safe_lsn
- FROM (pg_get_replication_slots() l(slot_name, plugin, slot_type, datoid, temporary, active, active_pid, xmin, catalog_xmin, restart_lsn, confirmed_flush_lsn, wal_status, min_safe_lsn)
+ l.distance
+ FROM (pg_get_replication_slots() l(slot_name, plugin, slot_type, datoid, temporary, active, active_pid, xmin, catalog_xmin, restart_lsn, confirmed_flush_lsn, wal_status, distance)
LEFT JOIN pg_database d ON ((l.datoid = d.oid)));
pg_roles| SELECT pg_authid.rolname,
pg_authid.rolsuper,
--
2.18.4
----Next_Part(Wed_Jul__1_10_32_59_2020_161)----
^ permalink raw reply [nested|flat] 32+ messages in thread
* [PATCH v1] Replace min_safe_lsn with "distance" in pg_replication_slots
@ 2020-06-30 12:09 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 32+ messages in thread
From: Kyotaro Horiguchi @ 2020-06-30 12:09 UTC (permalink / raw)
pg_replication_slot.min_safe_lsn, which shows the oldest LSN kept in
pg_wal, is doubtful in usability for monitoring. Change it to
distance, which shows how many bytes the server can advance before the
slot loses required segments.
---
src/backend/access/transam/xlog.c | 39 +++++++++++++++++++++++
src/backend/catalog/system_views.sql | 2 +-
src/backend/replication/slotfuncs.c | 19 +++++------
src/include/access/xlog.h | 1 +
src/include/catalog/pg_proc.dat | 4 +--
src/test/recovery/t/019_replslot_limit.pl | 20 ++++++------
src/test/regress/expected/rules.out | 4 +--
7 files changed, 62 insertions(+), 27 deletions(-)
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index fd93bcfaeb..1f27639912 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -9570,6 +9570,45 @@ GetWALAvailability(XLogRecPtr targetLSN)
return WALAVAIL_REMOVED;
}
+/*
+ * Calculate how many bytes we can advance from currptr until the targetLSN is
+ * removed.
+ *
+ * Returns 0 if the distance is invalid.
+ */
+uint64
+DistanceToWALHorizon(XLogRecPtr targetLSN, XLogRecPtr currptr)
+{
+ XLogSegNo targetSeg;
+ XLogSegNo keepSegs;
+ XLogSegNo failSeg;
+ XLogRecPtr horizon;
+
+ XLByteToSeg(targetLSN, targetSeg, wal_segment_size);
+ keepSegs = 0;
+
+ /* no limit if max_slot_wal_keep_size is invalid */
+ if (max_slot_wal_keep_size_mb < 0)
+ return 0;
+
+ /* How many segments slots can keep? */
+ keepSegs = ConvertToXSegs(max_slot_wal_keep_size_mb, wal_segment_size);
+
+ /* override by wal_keep_segments if needed */
+ if (wal_keep_segments > keepSegs)
+ keepSegs = wal_keep_segments;
+
+ /* calculate the LSN where targetLSN is lost when currpos reaches */
+ failSeg = targetSeg + keepSegs + 1;
+ XLogSegNoOffsetToRecPtr(failSeg, 0, wal_segment_size, horizon);
+
+ /* If currptr already beyond the horizon, return zero. */
+ if (currptr > horizon)
+ return 0;
+
+ /* return the distance from currptr to the horizon */
+ return horizon - currptr;
+}
/*
* Retreat *logSegNo to the last segment that we need to retain because of
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index 5314e9348f..b9847a9f92 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -879,7 +879,7 @@ CREATE VIEW pg_replication_slots AS
L.restart_lsn,
L.confirmed_flush_lsn,
L.wal_status,
- L.min_safe_lsn
+ L.distance
FROM pg_get_replication_slots() AS L
LEFT JOIN pg_database D ON (L.datoid = D.oid);
diff --git a/src/backend/replication/slotfuncs.c b/src/backend/replication/slotfuncs.c
index 88033a79b2..532b3c5826 100644
--- a/src/backend/replication/slotfuncs.c
+++ b/src/backend/replication/slotfuncs.c
@@ -242,6 +242,7 @@ pg_get_replication_slots(PG_FUNCTION_ARGS)
Tuplestorestate *tupstore;
MemoryContext per_query_ctx;
MemoryContext oldcontext;
+ XLogRecPtr currlsn;
int slotno;
/* check to see if caller supports us returning a tuplestore */
@@ -274,6 +275,8 @@ pg_get_replication_slots(PG_FUNCTION_ARGS)
MemoryContextSwitchTo(oldcontext);
+ currlsn = GetXLogWriteRecPtr();
+
LWLockAcquire(ReplicationSlotControlLock, LW_SHARED);
for (slotno = 0; slotno < max_replication_slots; slotno++)
{
@@ -282,7 +285,7 @@ pg_get_replication_slots(PG_FUNCTION_ARGS)
Datum values[PG_GET_REPLICATION_SLOTS_COLS];
bool nulls[PG_GET_REPLICATION_SLOTS_COLS];
WALAvailability walstate;
- XLogSegNo last_removed_seg;
+ uint32 distance;
int i;
if (!slot->in_use)
@@ -398,16 +401,10 @@ pg_get_replication_slots(PG_FUNCTION_ARGS)
break;
}
- if (max_slot_wal_keep_size_mb >= 0 &&
- (walstate == WALAVAIL_RESERVED || walstate == WALAVAIL_EXTENDED) &&
- ((last_removed_seg = XLogGetLastRemovedSegno()) != 0))
- {
- XLogRecPtr min_safe_lsn;
-
- XLogSegNoOffsetToRecPtr(last_removed_seg + 1, 0,
- wal_segment_size, min_safe_lsn);
- values[i++] = Int64GetDatum(min_safe_lsn);
- }
+ distance =
+ DistanceToWALHorizon(slot_contents.data.restart_lsn, currlsn);
+ if (distance > 0)
+ values[i++] = Int64GetDatum(distance);
else
nulls[i++] = true;
diff --git a/src/include/access/xlog.h b/src/include/access/xlog.h
index 77ac4e785f..1ec448c5d5 100644
--- a/src/include/access/xlog.h
+++ b/src/include/access/xlog.h
@@ -329,6 +329,7 @@ extern void InitXLOGAccess(void);
extern void CreateCheckPoint(int flags);
extern bool CreateRestartPoint(int flags);
extern WALAvailability GetWALAvailability(XLogRecPtr targetLSN);
+extern uint64 DistanceToWALHorizon(XLogRecPtr targetLSN, XLogRecPtr currptr);
extern XLogRecPtr CalculateMaxmumSafeLSN(void);
extern void XLogPutNextOid(Oid nextOid);
extern XLogRecPtr XLogRestorePoint(const char *rpName);
diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat
index 61f2c2f5b4..199fd994bd 100644
--- a/src/include/catalog/pg_proc.dat
+++ b/src/include/catalog/pg_proc.dat
@@ -10063,9 +10063,9 @@
proname => 'pg_get_replication_slots', prorows => '10', proisstrict => 'f',
proretset => 't', provolatile => 's', prorettype => 'record',
proargtypes => '',
- proallargtypes => '{name,name,text,oid,bool,bool,int4,xid,xid,pg_lsn,pg_lsn,text,pg_lsn}',
+ proallargtypes => '{name,name,text,oid,bool,bool,int4,xid,xid,pg_lsn,pg_lsn,text,int8}',
proargmodes => '{o,o,o,o,o,o,o,o,o,o,o,o,o}',
- proargnames => '{slot_name,plugin,slot_type,datoid,temporary,active,active_pid,xmin,catalog_xmin,restart_lsn,confirmed_flush_lsn,wal_status,min_safe_lsn}',
+ proargnames => '{slot_name,plugin,slot_type,datoid,temporary,active,active_pid,xmin,catalog_xmin,restart_lsn,confirmed_flush_lsn,wal_status,distance}',
prosrc => 'pg_get_replication_slots' },
{ oid => '3786', descr => 'set up a logical replication slot',
proname => 'pg_create_logical_replication_slot', provolatile => 'v',
diff --git a/src/test/recovery/t/019_replslot_limit.pl b/src/test/recovery/t/019_replslot_limit.pl
index 7d22ae5720..1c76d2d9e9 100644
--- a/src/test/recovery/t/019_replslot_limit.pl
+++ b/src/test/recovery/t/019_replslot_limit.pl
@@ -28,7 +28,7 @@ $node_master->safe_psql('postgres',
# The slot state and remain should be null before the first connection
my $result = $node_master->safe_psql('postgres',
- "SELECT restart_lsn IS NULL, wal_status is NULL, min_safe_lsn is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
+ "SELECT restart_lsn IS NULL, wal_status is NULL, distance is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
);
is($result, "t|t|t", 'check the state of non-reserved slot is "unknown"');
@@ -52,9 +52,9 @@ $node_master->wait_for_catchup($node_standby, 'replay', $start_lsn);
# Stop standby
$node_standby->stop;
-# Preparation done, the slot is the state "normal" now
+# Preparation done, the slot is the state "reserved" now
$result = $node_master->safe_psql('postgres',
- "SELECT wal_status, min_safe_lsn is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
+ "SELECT wal_status, distance is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
);
is($result, "reserved|t", 'check the catching-up state');
@@ -64,7 +64,7 @@ $node_master->safe_psql('postgres', "CHECKPOINT;");
# The slot is always "safe" when fitting max_wal_size
$result = $node_master->safe_psql('postgres',
- "SELECT wal_status, min_safe_lsn is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
+ "SELECT wal_status, distance is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
);
is($result, "reserved|t",
'check that it is safe if WAL fits in max_wal_size');
@@ -74,7 +74,7 @@ $node_master->safe_psql('postgres', "CHECKPOINT;");
# The slot is always "safe" when max_slot_wal_keep_size is not set
$result = $node_master->safe_psql('postgres',
- "SELECT wal_status, min_safe_lsn is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
+ "SELECT wal_status, distance is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
);
is($result, "reserved|t", 'check that slot is working');
@@ -94,9 +94,7 @@ max_slot_wal_keep_size = ${max_slot_wal_keep_size_mb}MB
));
$node_master->reload;
-# The slot is in safe state. The distance from the min_safe_lsn should
-# be as almost (max_slot_wal_keep_size - 1) times large as the segment
-# size
+# The slot is in safe state.
$result = $node_master->safe_psql('postgres',
"SELECT wal_status FROM pg_replication_slots WHERE slot_name = 'rep1'");
@@ -110,7 +108,7 @@ $node_master->safe_psql('postgres', "CHECKPOINT;");
$result = $node_master->safe_psql('postgres',
"SELECT wal_status FROM pg_replication_slots WHERE slot_name = 'rep1'");
is($result, "reserved",
- 'check that min_safe_lsn gets close to the current LSN');
+ 'check that distance gets close to the current LSN');
# The standby can reconnect to master
$node_standby->start;
@@ -154,7 +152,7 @@ advance_wal($node_master, 1);
# Slot gets into 'unreserved' state
$result = $node_master->safe_psql('postgres',
- "SELECT wal_status, min_safe_lsn is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
+ "SELECT wal_status, distance is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
);
is($result, "unreserved|t",
'check that the slot state changes to "unreserved"');
@@ -186,7 +184,7 @@ ok( find_in_log(
# This slot should be broken
$result = $node_master->safe_psql('postgres',
- "SELECT slot_name, active, restart_lsn IS NULL, wal_status, min_safe_lsn FROM pg_replication_slots WHERE slot_name = 'rep1'"
+ "SELECT slot_name, active, restart_lsn IS NULL, wal_status, distance FROM pg_replication_slots WHERE slot_name = 'rep1'"
);
is($result, "rep1|f|t|lost|",
'check that the slot became inactive and the state "lost" persists');
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b813e32215..392eab12dd 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1464,8 +1464,8 @@ pg_replication_slots| SELECT l.slot_name,
l.restart_lsn,
l.confirmed_flush_lsn,
l.wal_status,
- l.min_safe_lsn
- FROM (pg_get_replication_slots() l(slot_name, plugin, slot_type, datoid, temporary, active, active_pid, xmin, catalog_xmin, restart_lsn, confirmed_flush_lsn, wal_status, min_safe_lsn)
+ l.distance
+ FROM (pg_get_replication_slots() l(slot_name, plugin, slot_type, datoid, temporary, active, active_pid, xmin, catalog_xmin, restart_lsn, confirmed_flush_lsn, wal_status, distance)
LEFT JOIN pg_database d ON ((l.datoid = d.oid)));
pg_roles| SELECT pg_authid.rolname,
pg_authid.rolsuper,
--
2.18.4
----Next_Part(Wed_Jul__1_10_32_59_2020_161)----
^ permalink raw reply [nested|flat] 32+ messages in thread
* [PATCH v1] Replace min_safe_lsn with "distance" in pg_replication_slots
@ 2020-06-30 12:09 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 32+ messages in thread
From: Kyotaro Horiguchi @ 2020-06-30 12:09 UTC (permalink / raw)
pg_replication_slot.min_safe_lsn, which shows the oldest LSN kept in
pg_wal, is doubtful in usability for monitoring. Change it to
distance, which shows how many bytes the server can advance before the
slot loses required segments.
---
src/backend/access/transam/xlog.c | 39 +++++++++++++++++++++++
src/backend/catalog/system_views.sql | 2 +-
src/backend/replication/slotfuncs.c | 19 +++++------
src/include/access/xlog.h | 1 +
src/include/catalog/pg_proc.dat | 4 +--
src/test/recovery/t/019_replslot_limit.pl | 20 ++++++------
src/test/regress/expected/rules.out | 4 +--
7 files changed, 62 insertions(+), 27 deletions(-)
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index fd93bcfaeb..1f27639912 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -9570,6 +9570,45 @@ GetWALAvailability(XLogRecPtr targetLSN)
return WALAVAIL_REMOVED;
}
+/*
+ * Calculate how many bytes we can advance from currptr until the targetLSN is
+ * removed.
+ *
+ * Returns 0 if the distance is invalid.
+ */
+uint64
+DistanceToWALHorizon(XLogRecPtr targetLSN, XLogRecPtr currptr)
+{
+ XLogSegNo targetSeg;
+ XLogSegNo keepSegs;
+ XLogSegNo failSeg;
+ XLogRecPtr horizon;
+
+ XLByteToSeg(targetLSN, targetSeg, wal_segment_size);
+ keepSegs = 0;
+
+ /* no limit if max_slot_wal_keep_size is invalid */
+ if (max_slot_wal_keep_size_mb < 0)
+ return 0;
+
+ /* How many segments slots can keep? */
+ keepSegs = ConvertToXSegs(max_slot_wal_keep_size_mb, wal_segment_size);
+
+ /* override by wal_keep_segments if needed */
+ if (wal_keep_segments > keepSegs)
+ keepSegs = wal_keep_segments;
+
+ /* calculate the LSN where targetLSN is lost when currpos reaches */
+ failSeg = targetSeg + keepSegs + 1;
+ XLogSegNoOffsetToRecPtr(failSeg, 0, wal_segment_size, horizon);
+
+ /* If currptr already beyond the horizon, return zero. */
+ if (currptr > horizon)
+ return 0;
+
+ /* return the distance from currptr to the horizon */
+ return horizon - currptr;
+}
/*
* Retreat *logSegNo to the last segment that we need to retain because of
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index 5314e9348f..b9847a9f92 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -879,7 +879,7 @@ CREATE VIEW pg_replication_slots AS
L.restart_lsn,
L.confirmed_flush_lsn,
L.wal_status,
- L.min_safe_lsn
+ L.distance
FROM pg_get_replication_slots() AS L
LEFT JOIN pg_database D ON (L.datoid = D.oid);
diff --git a/src/backend/replication/slotfuncs.c b/src/backend/replication/slotfuncs.c
index 88033a79b2..532b3c5826 100644
--- a/src/backend/replication/slotfuncs.c
+++ b/src/backend/replication/slotfuncs.c
@@ -242,6 +242,7 @@ pg_get_replication_slots(PG_FUNCTION_ARGS)
Tuplestorestate *tupstore;
MemoryContext per_query_ctx;
MemoryContext oldcontext;
+ XLogRecPtr currlsn;
int slotno;
/* check to see if caller supports us returning a tuplestore */
@@ -274,6 +275,8 @@ pg_get_replication_slots(PG_FUNCTION_ARGS)
MemoryContextSwitchTo(oldcontext);
+ currlsn = GetXLogWriteRecPtr();
+
LWLockAcquire(ReplicationSlotControlLock, LW_SHARED);
for (slotno = 0; slotno < max_replication_slots; slotno++)
{
@@ -282,7 +285,7 @@ pg_get_replication_slots(PG_FUNCTION_ARGS)
Datum values[PG_GET_REPLICATION_SLOTS_COLS];
bool nulls[PG_GET_REPLICATION_SLOTS_COLS];
WALAvailability walstate;
- XLogSegNo last_removed_seg;
+ uint32 distance;
int i;
if (!slot->in_use)
@@ -398,16 +401,10 @@ pg_get_replication_slots(PG_FUNCTION_ARGS)
break;
}
- if (max_slot_wal_keep_size_mb >= 0 &&
- (walstate == WALAVAIL_RESERVED || walstate == WALAVAIL_EXTENDED) &&
- ((last_removed_seg = XLogGetLastRemovedSegno()) != 0))
- {
- XLogRecPtr min_safe_lsn;
-
- XLogSegNoOffsetToRecPtr(last_removed_seg + 1, 0,
- wal_segment_size, min_safe_lsn);
- values[i++] = Int64GetDatum(min_safe_lsn);
- }
+ distance =
+ DistanceToWALHorizon(slot_contents.data.restart_lsn, currlsn);
+ if (distance > 0)
+ values[i++] = Int64GetDatum(distance);
else
nulls[i++] = true;
diff --git a/src/include/access/xlog.h b/src/include/access/xlog.h
index 77ac4e785f..1ec448c5d5 100644
--- a/src/include/access/xlog.h
+++ b/src/include/access/xlog.h
@@ -329,6 +329,7 @@ extern void InitXLOGAccess(void);
extern void CreateCheckPoint(int flags);
extern bool CreateRestartPoint(int flags);
extern WALAvailability GetWALAvailability(XLogRecPtr targetLSN);
+extern uint64 DistanceToWALHorizon(XLogRecPtr targetLSN, XLogRecPtr currptr);
extern XLogRecPtr CalculateMaxmumSafeLSN(void);
extern void XLogPutNextOid(Oid nextOid);
extern XLogRecPtr XLogRestorePoint(const char *rpName);
diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat
index 61f2c2f5b4..199fd994bd 100644
--- a/src/include/catalog/pg_proc.dat
+++ b/src/include/catalog/pg_proc.dat
@@ -10063,9 +10063,9 @@
proname => 'pg_get_replication_slots', prorows => '10', proisstrict => 'f',
proretset => 't', provolatile => 's', prorettype => 'record',
proargtypes => '',
- proallargtypes => '{name,name,text,oid,bool,bool,int4,xid,xid,pg_lsn,pg_lsn,text,pg_lsn}',
+ proallargtypes => '{name,name,text,oid,bool,bool,int4,xid,xid,pg_lsn,pg_lsn,text,int8}',
proargmodes => '{o,o,o,o,o,o,o,o,o,o,o,o,o}',
- proargnames => '{slot_name,plugin,slot_type,datoid,temporary,active,active_pid,xmin,catalog_xmin,restart_lsn,confirmed_flush_lsn,wal_status,min_safe_lsn}',
+ proargnames => '{slot_name,plugin,slot_type,datoid,temporary,active,active_pid,xmin,catalog_xmin,restart_lsn,confirmed_flush_lsn,wal_status,distance}',
prosrc => 'pg_get_replication_slots' },
{ oid => '3786', descr => 'set up a logical replication slot',
proname => 'pg_create_logical_replication_slot', provolatile => 'v',
diff --git a/src/test/recovery/t/019_replslot_limit.pl b/src/test/recovery/t/019_replslot_limit.pl
index 7d22ae5720..1c76d2d9e9 100644
--- a/src/test/recovery/t/019_replslot_limit.pl
+++ b/src/test/recovery/t/019_replslot_limit.pl
@@ -28,7 +28,7 @@ $node_master->safe_psql('postgres',
# The slot state and remain should be null before the first connection
my $result = $node_master->safe_psql('postgres',
- "SELECT restart_lsn IS NULL, wal_status is NULL, min_safe_lsn is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
+ "SELECT restart_lsn IS NULL, wal_status is NULL, distance is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
);
is($result, "t|t|t", 'check the state of non-reserved slot is "unknown"');
@@ -52,9 +52,9 @@ $node_master->wait_for_catchup($node_standby, 'replay', $start_lsn);
# Stop standby
$node_standby->stop;
-# Preparation done, the slot is the state "normal" now
+# Preparation done, the slot is the state "reserved" now
$result = $node_master->safe_psql('postgres',
- "SELECT wal_status, min_safe_lsn is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
+ "SELECT wal_status, distance is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
);
is($result, "reserved|t", 'check the catching-up state');
@@ -64,7 +64,7 @@ $node_master->safe_psql('postgres', "CHECKPOINT;");
# The slot is always "safe" when fitting max_wal_size
$result = $node_master->safe_psql('postgres',
- "SELECT wal_status, min_safe_lsn is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
+ "SELECT wal_status, distance is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
);
is($result, "reserved|t",
'check that it is safe if WAL fits in max_wal_size');
@@ -74,7 +74,7 @@ $node_master->safe_psql('postgres', "CHECKPOINT;");
# The slot is always "safe" when max_slot_wal_keep_size is not set
$result = $node_master->safe_psql('postgres',
- "SELECT wal_status, min_safe_lsn is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
+ "SELECT wal_status, distance is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
);
is($result, "reserved|t", 'check that slot is working');
@@ -94,9 +94,7 @@ max_slot_wal_keep_size = ${max_slot_wal_keep_size_mb}MB
));
$node_master->reload;
-# The slot is in safe state. The distance from the min_safe_lsn should
-# be as almost (max_slot_wal_keep_size - 1) times large as the segment
-# size
+# The slot is in safe state.
$result = $node_master->safe_psql('postgres',
"SELECT wal_status FROM pg_replication_slots WHERE slot_name = 'rep1'");
@@ -110,7 +108,7 @@ $node_master->safe_psql('postgres', "CHECKPOINT;");
$result = $node_master->safe_psql('postgres',
"SELECT wal_status FROM pg_replication_slots WHERE slot_name = 'rep1'");
is($result, "reserved",
- 'check that min_safe_lsn gets close to the current LSN');
+ 'check that distance gets close to the current LSN');
# The standby can reconnect to master
$node_standby->start;
@@ -154,7 +152,7 @@ advance_wal($node_master, 1);
# Slot gets into 'unreserved' state
$result = $node_master->safe_psql('postgres',
- "SELECT wal_status, min_safe_lsn is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
+ "SELECT wal_status, distance is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
);
is($result, "unreserved|t",
'check that the slot state changes to "unreserved"');
@@ -186,7 +184,7 @@ ok( find_in_log(
# This slot should be broken
$result = $node_master->safe_psql('postgres',
- "SELECT slot_name, active, restart_lsn IS NULL, wal_status, min_safe_lsn FROM pg_replication_slots WHERE slot_name = 'rep1'"
+ "SELECT slot_name, active, restart_lsn IS NULL, wal_status, distance FROM pg_replication_slots WHERE slot_name = 'rep1'"
);
is($result, "rep1|f|t|lost|",
'check that the slot became inactive and the state "lost" persists');
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b813e32215..392eab12dd 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1464,8 +1464,8 @@ pg_replication_slots| SELECT l.slot_name,
l.restart_lsn,
l.confirmed_flush_lsn,
l.wal_status,
- l.min_safe_lsn
- FROM (pg_get_replication_slots() l(slot_name, plugin, slot_type, datoid, temporary, active, active_pid, xmin, catalog_xmin, restart_lsn, confirmed_flush_lsn, wal_status, min_safe_lsn)
+ l.distance
+ FROM (pg_get_replication_slots() l(slot_name, plugin, slot_type, datoid, temporary, active, active_pid, xmin, catalog_xmin, restart_lsn, confirmed_flush_lsn, wal_status, distance)
LEFT JOIN pg_database d ON ((l.datoid = d.oid)));
pg_roles| SELECT pg_authid.rolname,
pg_authid.rolsuper,
--
2.18.4
----Next_Part(Wed_Jul__1_10_32_59_2020_161)----
^ permalink raw reply [nested|flat] 32+ messages in thread
* [PATCH v1] Replace min_safe_lsn with "distance" in pg_replication_slots
@ 2020-06-30 12:09 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 32+ messages in thread
From: Kyotaro Horiguchi @ 2020-06-30 12:09 UTC (permalink / raw)
pg_replication_slot.min_safe_lsn, which shows the oldest LSN kept in
pg_wal, is doubtful in usability for monitoring. Change it to
distance, which shows how many bytes the server can advance before the
slot loses required segments.
---
src/backend/access/transam/xlog.c | 39 +++++++++++++++++++++++
src/backend/catalog/system_views.sql | 2 +-
src/backend/replication/slotfuncs.c | 19 +++++------
src/include/access/xlog.h | 1 +
src/include/catalog/pg_proc.dat | 4 +--
src/test/recovery/t/019_replslot_limit.pl | 20 ++++++------
src/test/regress/expected/rules.out | 4 +--
7 files changed, 62 insertions(+), 27 deletions(-)
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index fd93bcfaeb..1f27639912 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -9570,6 +9570,45 @@ GetWALAvailability(XLogRecPtr targetLSN)
return WALAVAIL_REMOVED;
}
+/*
+ * Calculate how many bytes we can advance from currptr until the targetLSN is
+ * removed.
+ *
+ * Returns 0 if the distance is invalid.
+ */
+uint64
+DistanceToWALHorizon(XLogRecPtr targetLSN, XLogRecPtr currptr)
+{
+ XLogSegNo targetSeg;
+ XLogSegNo keepSegs;
+ XLogSegNo failSeg;
+ XLogRecPtr horizon;
+
+ XLByteToSeg(targetLSN, targetSeg, wal_segment_size);
+ keepSegs = 0;
+
+ /* no limit if max_slot_wal_keep_size is invalid */
+ if (max_slot_wal_keep_size_mb < 0)
+ return 0;
+
+ /* How many segments slots can keep? */
+ keepSegs = ConvertToXSegs(max_slot_wal_keep_size_mb, wal_segment_size);
+
+ /* override by wal_keep_segments if needed */
+ if (wal_keep_segments > keepSegs)
+ keepSegs = wal_keep_segments;
+
+ /* calculate the LSN where targetLSN is lost when currpos reaches */
+ failSeg = targetSeg + keepSegs + 1;
+ XLogSegNoOffsetToRecPtr(failSeg, 0, wal_segment_size, horizon);
+
+ /* If currptr already beyond the horizon, return zero. */
+ if (currptr > horizon)
+ return 0;
+
+ /* return the distance from currptr to the horizon */
+ return horizon - currptr;
+}
/*
* Retreat *logSegNo to the last segment that we need to retain because of
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index 5314e9348f..b9847a9f92 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -879,7 +879,7 @@ CREATE VIEW pg_replication_slots AS
L.restart_lsn,
L.confirmed_flush_lsn,
L.wal_status,
- L.min_safe_lsn
+ L.distance
FROM pg_get_replication_slots() AS L
LEFT JOIN pg_database D ON (L.datoid = D.oid);
diff --git a/src/backend/replication/slotfuncs.c b/src/backend/replication/slotfuncs.c
index 88033a79b2..532b3c5826 100644
--- a/src/backend/replication/slotfuncs.c
+++ b/src/backend/replication/slotfuncs.c
@@ -242,6 +242,7 @@ pg_get_replication_slots(PG_FUNCTION_ARGS)
Tuplestorestate *tupstore;
MemoryContext per_query_ctx;
MemoryContext oldcontext;
+ XLogRecPtr currlsn;
int slotno;
/* check to see if caller supports us returning a tuplestore */
@@ -274,6 +275,8 @@ pg_get_replication_slots(PG_FUNCTION_ARGS)
MemoryContextSwitchTo(oldcontext);
+ currlsn = GetXLogWriteRecPtr();
+
LWLockAcquire(ReplicationSlotControlLock, LW_SHARED);
for (slotno = 0; slotno < max_replication_slots; slotno++)
{
@@ -282,7 +285,7 @@ pg_get_replication_slots(PG_FUNCTION_ARGS)
Datum values[PG_GET_REPLICATION_SLOTS_COLS];
bool nulls[PG_GET_REPLICATION_SLOTS_COLS];
WALAvailability walstate;
- XLogSegNo last_removed_seg;
+ uint32 distance;
int i;
if (!slot->in_use)
@@ -398,16 +401,10 @@ pg_get_replication_slots(PG_FUNCTION_ARGS)
break;
}
- if (max_slot_wal_keep_size_mb >= 0 &&
- (walstate == WALAVAIL_RESERVED || walstate == WALAVAIL_EXTENDED) &&
- ((last_removed_seg = XLogGetLastRemovedSegno()) != 0))
- {
- XLogRecPtr min_safe_lsn;
-
- XLogSegNoOffsetToRecPtr(last_removed_seg + 1, 0,
- wal_segment_size, min_safe_lsn);
- values[i++] = Int64GetDatum(min_safe_lsn);
- }
+ distance =
+ DistanceToWALHorizon(slot_contents.data.restart_lsn, currlsn);
+ if (distance > 0)
+ values[i++] = Int64GetDatum(distance);
else
nulls[i++] = true;
diff --git a/src/include/access/xlog.h b/src/include/access/xlog.h
index 77ac4e785f..1ec448c5d5 100644
--- a/src/include/access/xlog.h
+++ b/src/include/access/xlog.h
@@ -329,6 +329,7 @@ extern void InitXLOGAccess(void);
extern void CreateCheckPoint(int flags);
extern bool CreateRestartPoint(int flags);
extern WALAvailability GetWALAvailability(XLogRecPtr targetLSN);
+extern uint64 DistanceToWALHorizon(XLogRecPtr targetLSN, XLogRecPtr currptr);
extern XLogRecPtr CalculateMaxmumSafeLSN(void);
extern void XLogPutNextOid(Oid nextOid);
extern XLogRecPtr XLogRestorePoint(const char *rpName);
diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat
index 61f2c2f5b4..199fd994bd 100644
--- a/src/include/catalog/pg_proc.dat
+++ b/src/include/catalog/pg_proc.dat
@@ -10063,9 +10063,9 @@
proname => 'pg_get_replication_slots', prorows => '10', proisstrict => 'f',
proretset => 't', provolatile => 's', prorettype => 'record',
proargtypes => '',
- proallargtypes => '{name,name,text,oid,bool,bool,int4,xid,xid,pg_lsn,pg_lsn,text,pg_lsn}',
+ proallargtypes => '{name,name,text,oid,bool,bool,int4,xid,xid,pg_lsn,pg_lsn,text,int8}',
proargmodes => '{o,o,o,o,o,o,o,o,o,o,o,o,o}',
- proargnames => '{slot_name,plugin,slot_type,datoid,temporary,active,active_pid,xmin,catalog_xmin,restart_lsn,confirmed_flush_lsn,wal_status,min_safe_lsn}',
+ proargnames => '{slot_name,plugin,slot_type,datoid,temporary,active,active_pid,xmin,catalog_xmin,restart_lsn,confirmed_flush_lsn,wal_status,distance}',
prosrc => 'pg_get_replication_slots' },
{ oid => '3786', descr => 'set up a logical replication slot',
proname => 'pg_create_logical_replication_slot', provolatile => 'v',
diff --git a/src/test/recovery/t/019_replslot_limit.pl b/src/test/recovery/t/019_replslot_limit.pl
index 7d22ae5720..1c76d2d9e9 100644
--- a/src/test/recovery/t/019_replslot_limit.pl
+++ b/src/test/recovery/t/019_replslot_limit.pl
@@ -28,7 +28,7 @@ $node_master->safe_psql('postgres',
# The slot state and remain should be null before the first connection
my $result = $node_master->safe_psql('postgres',
- "SELECT restart_lsn IS NULL, wal_status is NULL, min_safe_lsn is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
+ "SELECT restart_lsn IS NULL, wal_status is NULL, distance is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
);
is($result, "t|t|t", 'check the state of non-reserved slot is "unknown"');
@@ -52,9 +52,9 @@ $node_master->wait_for_catchup($node_standby, 'replay', $start_lsn);
# Stop standby
$node_standby->stop;
-# Preparation done, the slot is the state "normal" now
+# Preparation done, the slot is the state "reserved" now
$result = $node_master->safe_psql('postgres',
- "SELECT wal_status, min_safe_lsn is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
+ "SELECT wal_status, distance is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
);
is($result, "reserved|t", 'check the catching-up state');
@@ -64,7 +64,7 @@ $node_master->safe_psql('postgres', "CHECKPOINT;");
# The slot is always "safe" when fitting max_wal_size
$result = $node_master->safe_psql('postgres',
- "SELECT wal_status, min_safe_lsn is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
+ "SELECT wal_status, distance is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
);
is($result, "reserved|t",
'check that it is safe if WAL fits in max_wal_size');
@@ -74,7 +74,7 @@ $node_master->safe_psql('postgres', "CHECKPOINT;");
# The slot is always "safe" when max_slot_wal_keep_size is not set
$result = $node_master->safe_psql('postgres',
- "SELECT wal_status, min_safe_lsn is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
+ "SELECT wal_status, distance is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
);
is($result, "reserved|t", 'check that slot is working');
@@ -94,9 +94,7 @@ max_slot_wal_keep_size = ${max_slot_wal_keep_size_mb}MB
));
$node_master->reload;
-# The slot is in safe state. The distance from the min_safe_lsn should
-# be as almost (max_slot_wal_keep_size - 1) times large as the segment
-# size
+# The slot is in safe state.
$result = $node_master->safe_psql('postgres',
"SELECT wal_status FROM pg_replication_slots WHERE slot_name = 'rep1'");
@@ -110,7 +108,7 @@ $node_master->safe_psql('postgres', "CHECKPOINT;");
$result = $node_master->safe_psql('postgres',
"SELECT wal_status FROM pg_replication_slots WHERE slot_name = 'rep1'");
is($result, "reserved",
- 'check that min_safe_lsn gets close to the current LSN');
+ 'check that distance gets close to the current LSN');
# The standby can reconnect to master
$node_standby->start;
@@ -154,7 +152,7 @@ advance_wal($node_master, 1);
# Slot gets into 'unreserved' state
$result = $node_master->safe_psql('postgres',
- "SELECT wal_status, min_safe_lsn is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
+ "SELECT wal_status, distance is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
);
is($result, "unreserved|t",
'check that the slot state changes to "unreserved"');
@@ -186,7 +184,7 @@ ok( find_in_log(
# This slot should be broken
$result = $node_master->safe_psql('postgres',
- "SELECT slot_name, active, restart_lsn IS NULL, wal_status, min_safe_lsn FROM pg_replication_slots WHERE slot_name = 'rep1'"
+ "SELECT slot_name, active, restart_lsn IS NULL, wal_status, distance FROM pg_replication_slots WHERE slot_name = 'rep1'"
);
is($result, "rep1|f|t|lost|",
'check that the slot became inactive and the state "lost" persists');
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b813e32215..392eab12dd 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1464,8 +1464,8 @@ pg_replication_slots| SELECT l.slot_name,
l.restart_lsn,
l.confirmed_flush_lsn,
l.wal_status,
- l.min_safe_lsn
- FROM (pg_get_replication_slots() l(slot_name, plugin, slot_type, datoid, temporary, active, active_pid, xmin, catalog_xmin, restart_lsn, confirmed_flush_lsn, wal_status, min_safe_lsn)
+ l.distance
+ FROM (pg_get_replication_slots() l(slot_name, plugin, slot_type, datoid, temporary, active, active_pid, xmin, catalog_xmin, restart_lsn, confirmed_flush_lsn, wal_status, distance)
LEFT JOIN pg_database d ON ((l.datoid = d.oid)));
pg_roles| SELECT pg_authid.rolname,
pg_authid.rolsuper,
--
2.18.4
----Next_Part(Wed_Jul__1_10_32_59_2020_161)----
^ permalink raw reply [nested|flat] 32+ messages in thread
* [PATCH v1] Replace min_safe_lsn with "distance" in pg_replication_slots
@ 2020-06-30 12:09 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 32+ messages in thread
From: Kyotaro Horiguchi @ 2020-06-30 12:09 UTC (permalink / raw)
pg_replication_slot.min_safe_lsn, which shows the oldest LSN kept in
pg_wal, is doubtful in usability for monitoring. Change it to
distance, which shows how many bytes the server can advance before the
slot loses required segments.
---
src/backend/access/transam/xlog.c | 39 +++++++++++++++++++++++
src/backend/catalog/system_views.sql | 2 +-
src/backend/replication/slotfuncs.c | 19 +++++------
src/include/access/xlog.h | 1 +
src/include/catalog/pg_proc.dat | 4 +--
src/test/recovery/t/019_replslot_limit.pl | 20 ++++++------
src/test/regress/expected/rules.out | 4 +--
7 files changed, 62 insertions(+), 27 deletions(-)
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index fd93bcfaeb..1f27639912 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -9570,6 +9570,45 @@ GetWALAvailability(XLogRecPtr targetLSN)
return WALAVAIL_REMOVED;
}
+/*
+ * Calculate how many bytes we can advance from currptr until the targetLSN is
+ * removed.
+ *
+ * Returns 0 if the distance is invalid.
+ */
+uint64
+DistanceToWALHorizon(XLogRecPtr targetLSN, XLogRecPtr currptr)
+{
+ XLogSegNo targetSeg;
+ XLogSegNo keepSegs;
+ XLogSegNo failSeg;
+ XLogRecPtr horizon;
+
+ XLByteToSeg(targetLSN, targetSeg, wal_segment_size);
+ keepSegs = 0;
+
+ /* no limit if max_slot_wal_keep_size is invalid */
+ if (max_slot_wal_keep_size_mb < 0)
+ return 0;
+
+ /* How many segments slots can keep? */
+ keepSegs = ConvertToXSegs(max_slot_wal_keep_size_mb, wal_segment_size);
+
+ /* override by wal_keep_segments if needed */
+ if (wal_keep_segments > keepSegs)
+ keepSegs = wal_keep_segments;
+
+ /* calculate the LSN where targetLSN is lost when currpos reaches */
+ failSeg = targetSeg + keepSegs + 1;
+ XLogSegNoOffsetToRecPtr(failSeg, 0, wal_segment_size, horizon);
+
+ /* If currptr already beyond the horizon, return zero. */
+ if (currptr > horizon)
+ return 0;
+
+ /* return the distance from currptr to the horizon */
+ return horizon - currptr;
+}
/*
* Retreat *logSegNo to the last segment that we need to retain because of
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index 5314e9348f..b9847a9f92 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -879,7 +879,7 @@ CREATE VIEW pg_replication_slots AS
L.restart_lsn,
L.confirmed_flush_lsn,
L.wal_status,
- L.min_safe_lsn
+ L.distance
FROM pg_get_replication_slots() AS L
LEFT JOIN pg_database D ON (L.datoid = D.oid);
diff --git a/src/backend/replication/slotfuncs.c b/src/backend/replication/slotfuncs.c
index 88033a79b2..532b3c5826 100644
--- a/src/backend/replication/slotfuncs.c
+++ b/src/backend/replication/slotfuncs.c
@@ -242,6 +242,7 @@ pg_get_replication_slots(PG_FUNCTION_ARGS)
Tuplestorestate *tupstore;
MemoryContext per_query_ctx;
MemoryContext oldcontext;
+ XLogRecPtr currlsn;
int slotno;
/* check to see if caller supports us returning a tuplestore */
@@ -274,6 +275,8 @@ pg_get_replication_slots(PG_FUNCTION_ARGS)
MemoryContextSwitchTo(oldcontext);
+ currlsn = GetXLogWriteRecPtr();
+
LWLockAcquire(ReplicationSlotControlLock, LW_SHARED);
for (slotno = 0; slotno < max_replication_slots; slotno++)
{
@@ -282,7 +285,7 @@ pg_get_replication_slots(PG_FUNCTION_ARGS)
Datum values[PG_GET_REPLICATION_SLOTS_COLS];
bool nulls[PG_GET_REPLICATION_SLOTS_COLS];
WALAvailability walstate;
- XLogSegNo last_removed_seg;
+ uint32 distance;
int i;
if (!slot->in_use)
@@ -398,16 +401,10 @@ pg_get_replication_slots(PG_FUNCTION_ARGS)
break;
}
- if (max_slot_wal_keep_size_mb >= 0 &&
- (walstate == WALAVAIL_RESERVED || walstate == WALAVAIL_EXTENDED) &&
- ((last_removed_seg = XLogGetLastRemovedSegno()) != 0))
- {
- XLogRecPtr min_safe_lsn;
-
- XLogSegNoOffsetToRecPtr(last_removed_seg + 1, 0,
- wal_segment_size, min_safe_lsn);
- values[i++] = Int64GetDatum(min_safe_lsn);
- }
+ distance =
+ DistanceToWALHorizon(slot_contents.data.restart_lsn, currlsn);
+ if (distance > 0)
+ values[i++] = Int64GetDatum(distance);
else
nulls[i++] = true;
diff --git a/src/include/access/xlog.h b/src/include/access/xlog.h
index 77ac4e785f..1ec448c5d5 100644
--- a/src/include/access/xlog.h
+++ b/src/include/access/xlog.h
@@ -329,6 +329,7 @@ extern void InitXLOGAccess(void);
extern void CreateCheckPoint(int flags);
extern bool CreateRestartPoint(int flags);
extern WALAvailability GetWALAvailability(XLogRecPtr targetLSN);
+extern uint64 DistanceToWALHorizon(XLogRecPtr targetLSN, XLogRecPtr currptr);
extern XLogRecPtr CalculateMaxmumSafeLSN(void);
extern void XLogPutNextOid(Oid nextOid);
extern XLogRecPtr XLogRestorePoint(const char *rpName);
diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat
index 61f2c2f5b4..199fd994bd 100644
--- a/src/include/catalog/pg_proc.dat
+++ b/src/include/catalog/pg_proc.dat
@@ -10063,9 +10063,9 @@
proname => 'pg_get_replication_slots', prorows => '10', proisstrict => 'f',
proretset => 't', provolatile => 's', prorettype => 'record',
proargtypes => '',
- proallargtypes => '{name,name,text,oid,bool,bool,int4,xid,xid,pg_lsn,pg_lsn,text,pg_lsn}',
+ proallargtypes => '{name,name,text,oid,bool,bool,int4,xid,xid,pg_lsn,pg_lsn,text,int8}',
proargmodes => '{o,o,o,o,o,o,o,o,o,o,o,o,o}',
- proargnames => '{slot_name,plugin,slot_type,datoid,temporary,active,active_pid,xmin,catalog_xmin,restart_lsn,confirmed_flush_lsn,wal_status,min_safe_lsn}',
+ proargnames => '{slot_name,plugin,slot_type,datoid,temporary,active,active_pid,xmin,catalog_xmin,restart_lsn,confirmed_flush_lsn,wal_status,distance}',
prosrc => 'pg_get_replication_slots' },
{ oid => '3786', descr => 'set up a logical replication slot',
proname => 'pg_create_logical_replication_slot', provolatile => 'v',
diff --git a/src/test/recovery/t/019_replslot_limit.pl b/src/test/recovery/t/019_replslot_limit.pl
index 7d22ae5720..1c76d2d9e9 100644
--- a/src/test/recovery/t/019_replslot_limit.pl
+++ b/src/test/recovery/t/019_replslot_limit.pl
@@ -28,7 +28,7 @@ $node_master->safe_psql('postgres',
# The slot state and remain should be null before the first connection
my $result = $node_master->safe_psql('postgres',
- "SELECT restart_lsn IS NULL, wal_status is NULL, min_safe_lsn is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
+ "SELECT restart_lsn IS NULL, wal_status is NULL, distance is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
);
is($result, "t|t|t", 'check the state of non-reserved slot is "unknown"');
@@ -52,9 +52,9 @@ $node_master->wait_for_catchup($node_standby, 'replay', $start_lsn);
# Stop standby
$node_standby->stop;
-# Preparation done, the slot is the state "normal" now
+# Preparation done, the slot is the state "reserved" now
$result = $node_master->safe_psql('postgres',
- "SELECT wal_status, min_safe_lsn is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
+ "SELECT wal_status, distance is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
);
is($result, "reserved|t", 'check the catching-up state');
@@ -64,7 +64,7 @@ $node_master->safe_psql('postgres', "CHECKPOINT;");
# The slot is always "safe" when fitting max_wal_size
$result = $node_master->safe_psql('postgres',
- "SELECT wal_status, min_safe_lsn is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
+ "SELECT wal_status, distance is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
);
is($result, "reserved|t",
'check that it is safe if WAL fits in max_wal_size');
@@ -74,7 +74,7 @@ $node_master->safe_psql('postgres', "CHECKPOINT;");
# The slot is always "safe" when max_slot_wal_keep_size is not set
$result = $node_master->safe_psql('postgres',
- "SELECT wal_status, min_safe_lsn is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
+ "SELECT wal_status, distance is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
);
is($result, "reserved|t", 'check that slot is working');
@@ -94,9 +94,7 @@ max_slot_wal_keep_size = ${max_slot_wal_keep_size_mb}MB
));
$node_master->reload;
-# The slot is in safe state. The distance from the min_safe_lsn should
-# be as almost (max_slot_wal_keep_size - 1) times large as the segment
-# size
+# The slot is in safe state.
$result = $node_master->safe_psql('postgres',
"SELECT wal_status FROM pg_replication_slots WHERE slot_name = 'rep1'");
@@ -110,7 +108,7 @@ $node_master->safe_psql('postgres', "CHECKPOINT;");
$result = $node_master->safe_psql('postgres',
"SELECT wal_status FROM pg_replication_slots WHERE slot_name = 'rep1'");
is($result, "reserved",
- 'check that min_safe_lsn gets close to the current LSN');
+ 'check that distance gets close to the current LSN');
# The standby can reconnect to master
$node_standby->start;
@@ -154,7 +152,7 @@ advance_wal($node_master, 1);
# Slot gets into 'unreserved' state
$result = $node_master->safe_psql('postgres',
- "SELECT wal_status, min_safe_lsn is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
+ "SELECT wal_status, distance is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
);
is($result, "unreserved|t",
'check that the slot state changes to "unreserved"');
@@ -186,7 +184,7 @@ ok( find_in_log(
# This slot should be broken
$result = $node_master->safe_psql('postgres',
- "SELECT slot_name, active, restart_lsn IS NULL, wal_status, min_safe_lsn FROM pg_replication_slots WHERE slot_name = 'rep1'"
+ "SELECT slot_name, active, restart_lsn IS NULL, wal_status, distance FROM pg_replication_slots WHERE slot_name = 'rep1'"
);
is($result, "rep1|f|t|lost|",
'check that the slot became inactive and the state "lost" persists');
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b813e32215..392eab12dd 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1464,8 +1464,8 @@ pg_replication_slots| SELECT l.slot_name,
l.restart_lsn,
l.confirmed_flush_lsn,
l.wal_status,
- l.min_safe_lsn
- FROM (pg_get_replication_slots() l(slot_name, plugin, slot_type, datoid, temporary, active, active_pid, xmin, catalog_xmin, restart_lsn, confirmed_flush_lsn, wal_status, min_safe_lsn)
+ l.distance
+ FROM (pg_get_replication_slots() l(slot_name, plugin, slot_type, datoid, temporary, active, active_pid, xmin, catalog_xmin, restart_lsn, confirmed_flush_lsn, wal_status, distance)
LEFT JOIN pg_database d ON ((l.datoid = d.oid)));
pg_roles| SELECT pg_authid.rolname,
pg_authid.rolsuper,
--
2.18.4
----Next_Part(Wed_Jul__1_10_32_59_2020_161)----
^ permalink raw reply [nested|flat] 32+ messages in thread
* [PATCH v1] Replace min_safe_lsn with "distance" in pg_replication_slots
@ 2020-06-30 12:09 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 32+ messages in thread
From: Kyotaro Horiguchi @ 2020-06-30 12:09 UTC (permalink / raw)
pg_replication_slot.min_safe_lsn, which shows the oldest LSN kept in
pg_wal, is doubtful in usability for monitoring. Change it to
distance, which shows how many bytes the server can advance before the
slot loses required segments.
---
src/backend/access/transam/xlog.c | 39 +++++++++++++++++++++++
src/backend/catalog/system_views.sql | 2 +-
src/backend/replication/slotfuncs.c | 19 +++++------
src/include/access/xlog.h | 1 +
src/include/catalog/pg_proc.dat | 4 +--
src/test/recovery/t/019_replslot_limit.pl | 20 ++++++------
src/test/regress/expected/rules.out | 4 +--
7 files changed, 62 insertions(+), 27 deletions(-)
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index fd93bcfaeb..1f27639912 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -9570,6 +9570,45 @@ GetWALAvailability(XLogRecPtr targetLSN)
return WALAVAIL_REMOVED;
}
+/*
+ * Calculate how many bytes we can advance from currptr until the targetLSN is
+ * removed.
+ *
+ * Returns 0 if the distance is invalid.
+ */
+uint64
+DistanceToWALHorizon(XLogRecPtr targetLSN, XLogRecPtr currptr)
+{
+ XLogSegNo targetSeg;
+ XLogSegNo keepSegs;
+ XLogSegNo failSeg;
+ XLogRecPtr horizon;
+
+ XLByteToSeg(targetLSN, targetSeg, wal_segment_size);
+ keepSegs = 0;
+
+ /* no limit if max_slot_wal_keep_size is invalid */
+ if (max_slot_wal_keep_size_mb < 0)
+ return 0;
+
+ /* How many segments slots can keep? */
+ keepSegs = ConvertToXSegs(max_slot_wal_keep_size_mb, wal_segment_size);
+
+ /* override by wal_keep_segments if needed */
+ if (wal_keep_segments > keepSegs)
+ keepSegs = wal_keep_segments;
+
+ /* calculate the LSN where targetLSN is lost when currpos reaches */
+ failSeg = targetSeg + keepSegs + 1;
+ XLogSegNoOffsetToRecPtr(failSeg, 0, wal_segment_size, horizon);
+
+ /* If currptr already beyond the horizon, return zero. */
+ if (currptr > horizon)
+ return 0;
+
+ /* return the distance from currptr to the horizon */
+ return horizon - currptr;
+}
/*
* Retreat *logSegNo to the last segment that we need to retain because of
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index 5314e9348f..b9847a9f92 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -879,7 +879,7 @@ CREATE VIEW pg_replication_slots AS
L.restart_lsn,
L.confirmed_flush_lsn,
L.wal_status,
- L.min_safe_lsn
+ L.distance
FROM pg_get_replication_slots() AS L
LEFT JOIN pg_database D ON (L.datoid = D.oid);
diff --git a/src/backend/replication/slotfuncs.c b/src/backend/replication/slotfuncs.c
index 88033a79b2..532b3c5826 100644
--- a/src/backend/replication/slotfuncs.c
+++ b/src/backend/replication/slotfuncs.c
@@ -242,6 +242,7 @@ pg_get_replication_slots(PG_FUNCTION_ARGS)
Tuplestorestate *tupstore;
MemoryContext per_query_ctx;
MemoryContext oldcontext;
+ XLogRecPtr currlsn;
int slotno;
/* check to see if caller supports us returning a tuplestore */
@@ -274,6 +275,8 @@ pg_get_replication_slots(PG_FUNCTION_ARGS)
MemoryContextSwitchTo(oldcontext);
+ currlsn = GetXLogWriteRecPtr();
+
LWLockAcquire(ReplicationSlotControlLock, LW_SHARED);
for (slotno = 0; slotno < max_replication_slots; slotno++)
{
@@ -282,7 +285,7 @@ pg_get_replication_slots(PG_FUNCTION_ARGS)
Datum values[PG_GET_REPLICATION_SLOTS_COLS];
bool nulls[PG_GET_REPLICATION_SLOTS_COLS];
WALAvailability walstate;
- XLogSegNo last_removed_seg;
+ uint32 distance;
int i;
if (!slot->in_use)
@@ -398,16 +401,10 @@ pg_get_replication_slots(PG_FUNCTION_ARGS)
break;
}
- if (max_slot_wal_keep_size_mb >= 0 &&
- (walstate == WALAVAIL_RESERVED || walstate == WALAVAIL_EXTENDED) &&
- ((last_removed_seg = XLogGetLastRemovedSegno()) != 0))
- {
- XLogRecPtr min_safe_lsn;
-
- XLogSegNoOffsetToRecPtr(last_removed_seg + 1, 0,
- wal_segment_size, min_safe_lsn);
- values[i++] = Int64GetDatum(min_safe_lsn);
- }
+ distance =
+ DistanceToWALHorizon(slot_contents.data.restart_lsn, currlsn);
+ if (distance > 0)
+ values[i++] = Int64GetDatum(distance);
else
nulls[i++] = true;
diff --git a/src/include/access/xlog.h b/src/include/access/xlog.h
index 77ac4e785f..1ec448c5d5 100644
--- a/src/include/access/xlog.h
+++ b/src/include/access/xlog.h
@@ -329,6 +329,7 @@ extern void InitXLOGAccess(void);
extern void CreateCheckPoint(int flags);
extern bool CreateRestartPoint(int flags);
extern WALAvailability GetWALAvailability(XLogRecPtr targetLSN);
+extern uint64 DistanceToWALHorizon(XLogRecPtr targetLSN, XLogRecPtr currptr);
extern XLogRecPtr CalculateMaxmumSafeLSN(void);
extern void XLogPutNextOid(Oid nextOid);
extern XLogRecPtr XLogRestorePoint(const char *rpName);
diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat
index 61f2c2f5b4..199fd994bd 100644
--- a/src/include/catalog/pg_proc.dat
+++ b/src/include/catalog/pg_proc.dat
@@ -10063,9 +10063,9 @@
proname => 'pg_get_replication_slots', prorows => '10', proisstrict => 'f',
proretset => 't', provolatile => 's', prorettype => 'record',
proargtypes => '',
- proallargtypes => '{name,name,text,oid,bool,bool,int4,xid,xid,pg_lsn,pg_lsn,text,pg_lsn}',
+ proallargtypes => '{name,name,text,oid,bool,bool,int4,xid,xid,pg_lsn,pg_lsn,text,int8}',
proargmodes => '{o,o,o,o,o,o,o,o,o,o,o,o,o}',
- proargnames => '{slot_name,plugin,slot_type,datoid,temporary,active,active_pid,xmin,catalog_xmin,restart_lsn,confirmed_flush_lsn,wal_status,min_safe_lsn}',
+ proargnames => '{slot_name,plugin,slot_type,datoid,temporary,active,active_pid,xmin,catalog_xmin,restart_lsn,confirmed_flush_lsn,wal_status,distance}',
prosrc => 'pg_get_replication_slots' },
{ oid => '3786', descr => 'set up a logical replication slot',
proname => 'pg_create_logical_replication_slot', provolatile => 'v',
diff --git a/src/test/recovery/t/019_replslot_limit.pl b/src/test/recovery/t/019_replslot_limit.pl
index 7d22ae5720..1c76d2d9e9 100644
--- a/src/test/recovery/t/019_replslot_limit.pl
+++ b/src/test/recovery/t/019_replslot_limit.pl
@@ -28,7 +28,7 @@ $node_master->safe_psql('postgres',
# The slot state and remain should be null before the first connection
my $result = $node_master->safe_psql('postgres',
- "SELECT restart_lsn IS NULL, wal_status is NULL, min_safe_lsn is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
+ "SELECT restart_lsn IS NULL, wal_status is NULL, distance is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
);
is($result, "t|t|t", 'check the state of non-reserved slot is "unknown"');
@@ -52,9 +52,9 @@ $node_master->wait_for_catchup($node_standby, 'replay', $start_lsn);
# Stop standby
$node_standby->stop;
-# Preparation done, the slot is the state "normal" now
+# Preparation done, the slot is the state "reserved" now
$result = $node_master->safe_psql('postgres',
- "SELECT wal_status, min_safe_lsn is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
+ "SELECT wal_status, distance is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
);
is($result, "reserved|t", 'check the catching-up state');
@@ -64,7 +64,7 @@ $node_master->safe_psql('postgres', "CHECKPOINT;");
# The slot is always "safe" when fitting max_wal_size
$result = $node_master->safe_psql('postgres',
- "SELECT wal_status, min_safe_lsn is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
+ "SELECT wal_status, distance is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
);
is($result, "reserved|t",
'check that it is safe if WAL fits in max_wal_size');
@@ -74,7 +74,7 @@ $node_master->safe_psql('postgres', "CHECKPOINT;");
# The slot is always "safe" when max_slot_wal_keep_size is not set
$result = $node_master->safe_psql('postgres',
- "SELECT wal_status, min_safe_lsn is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
+ "SELECT wal_status, distance is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
);
is($result, "reserved|t", 'check that slot is working');
@@ -94,9 +94,7 @@ max_slot_wal_keep_size = ${max_slot_wal_keep_size_mb}MB
));
$node_master->reload;
-# The slot is in safe state. The distance from the min_safe_lsn should
-# be as almost (max_slot_wal_keep_size - 1) times large as the segment
-# size
+# The slot is in safe state.
$result = $node_master->safe_psql('postgres',
"SELECT wal_status FROM pg_replication_slots WHERE slot_name = 'rep1'");
@@ -110,7 +108,7 @@ $node_master->safe_psql('postgres', "CHECKPOINT;");
$result = $node_master->safe_psql('postgres',
"SELECT wal_status FROM pg_replication_slots WHERE slot_name = 'rep1'");
is($result, "reserved",
- 'check that min_safe_lsn gets close to the current LSN');
+ 'check that distance gets close to the current LSN');
# The standby can reconnect to master
$node_standby->start;
@@ -154,7 +152,7 @@ advance_wal($node_master, 1);
# Slot gets into 'unreserved' state
$result = $node_master->safe_psql('postgres',
- "SELECT wal_status, min_safe_lsn is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
+ "SELECT wal_status, distance is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
);
is($result, "unreserved|t",
'check that the slot state changes to "unreserved"');
@@ -186,7 +184,7 @@ ok( find_in_log(
# This slot should be broken
$result = $node_master->safe_psql('postgres',
- "SELECT slot_name, active, restart_lsn IS NULL, wal_status, min_safe_lsn FROM pg_replication_slots WHERE slot_name = 'rep1'"
+ "SELECT slot_name, active, restart_lsn IS NULL, wal_status, distance FROM pg_replication_slots WHERE slot_name = 'rep1'"
);
is($result, "rep1|f|t|lost|",
'check that the slot became inactive and the state "lost" persists');
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b813e32215..392eab12dd 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1464,8 +1464,8 @@ pg_replication_slots| SELECT l.slot_name,
l.restart_lsn,
l.confirmed_flush_lsn,
l.wal_status,
- l.min_safe_lsn
- FROM (pg_get_replication_slots() l(slot_name, plugin, slot_type, datoid, temporary, active, active_pid, xmin, catalog_xmin, restart_lsn, confirmed_flush_lsn, wal_status, min_safe_lsn)
+ l.distance
+ FROM (pg_get_replication_slots() l(slot_name, plugin, slot_type, datoid, temporary, active, active_pid, xmin, catalog_xmin, restart_lsn, confirmed_flush_lsn, wal_status, distance)
LEFT JOIN pg_database d ON ((l.datoid = d.oid)));
pg_roles| SELECT pg_authid.rolname,
pg_authid.rolsuper,
--
2.18.4
----Next_Part(Wed_Jul__1_10_32_59_2020_161)----
^ permalink raw reply [nested|flat] 32+ messages in thread
* [PATCH v1] Replace min_safe_lsn with "distance" in pg_replication_slots
@ 2020-06-30 12:09 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 32+ messages in thread
From: Kyotaro Horiguchi @ 2020-06-30 12:09 UTC (permalink / raw)
pg_replication_slot.min_safe_lsn, which shows the oldest LSN kept in
pg_wal, is doubtful in usability for monitoring. Change it to
distance, which shows how many bytes the server can advance before the
slot loses required segments.
---
src/backend/access/transam/xlog.c | 39 +++++++++++++++++++++++
src/backend/catalog/system_views.sql | 2 +-
src/backend/replication/slotfuncs.c | 19 +++++------
src/include/access/xlog.h | 1 +
src/include/catalog/pg_proc.dat | 4 +--
src/test/recovery/t/019_replslot_limit.pl | 20 ++++++------
src/test/regress/expected/rules.out | 4 +--
7 files changed, 62 insertions(+), 27 deletions(-)
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index fd93bcfaeb..1f27639912 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -9570,6 +9570,45 @@ GetWALAvailability(XLogRecPtr targetLSN)
return WALAVAIL_REMOVED;
}
+/*
+ * Calculate how many bytes we can advance from currptr until the targetLSN is
+ * removed.
+ *
+ * Returns 0 if the distance is invalid.
+ */
+uint64
+DistanceToWALHorizon(XLogRecPtr targetLSN, XLogRecPtr currptr)
+{
+ XLogSegNo targetSeg;
+ XLogSegNo keepSegs;
+ XLogSegNo failSeg;
+ XLogRecPtr horizon;
+
+ XLByteToSeg(targetLSN, targetSeg, wal_segment_size);
+ keepSegs = 0;
+
+ /* no limit if max_slot_wal_keep_size is invalid */
+ if (max_slot_wal_keep_size_mb < 0)
+ return 0;
+
+ /* How many segments slots can keep? */
+ keepSegs = ConvertToXSegs(max_slot_wal_keep_size_mb, wal_segment_size);
+
+ /* override by wal_keep_segments if needed */
+ if (wal_keep_segments > keepSegs)
+ keepSegs = wal_keep_segments;
+
+ /* calculate the LSN where targetLSN is lost when currpos reaches */
+ failSeg = targetSeg + keepSegs + 1;
+ XLogSegNoOffsetToRecPtr(failSeg, 0, wal_segment_size, horizon);
+
+ /* If currptr already beyond the horizon, return zero. */
+ if (currptr > horizon)
+ return 0;
+
+ /* return the distance from currptr to the horizon */
+ return horizon - currptr;
+}
/*
* Retreat *logSegNo to the last segment that we need to retain because of
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index 5314e9348f..b9847a9f92 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -879,7 +879,7 @@ CREATE VIEW pg_replication_slots AS
L.restart_lsn,
L.confirmed_flush_lsn,
L.wal_status,
- L.min_safe_lsn
+ L.distance
FROM pg_get_replication_slots() AS L
LEFT JOIN pg_database D ON (L.datoid = D.oid);
diff --git a/src/backend/replication/slotfuncs.c b/src/backend/replication/slotfuncs.c
index 88033a79b2..532b3c5826 100644
--- a/src/backend/replication/slotfuncs.c
+++ b/src/backend/replication/slotfuncs.c
@@ -242,6 +242,7 @@ pg_get_replication_slots(PG_FUNCTION_ARGS)
Tuplestorestate *tupstore;
MemoryContext per_query_ctx;
MemoryContext oldcontext;
+ XLogRecPtr currlsn;
int slotno;
/* check to see if caller supports us returning a tuplestore */
@@ -274,6 +275,8 @@ pg_get_replication_slots(PG_FUNCTION_ARGS)
MemoryContextSwitchTo(oldcontext);
+ currlsn = GetXLogWriteRecPtr();
+
LWLockAcquire(ReplicationSlotControlLock, LW_SHARED);
for (slotno = 0; slotno < max_replication_slots; slotno++)
{
@@ -282,7 +285,7 @@ pg_get_replication_slots(PG_FUNCTION_ARGS)
Datum values[PG_GET_REPLICATION_SLOTS_COLS];
bool nulls[PG_GET_REPLICATION_SLOTS_COLS];
WALAvailability walstate;
- XLogSegNo last_removed_seg;
+ uint32 distance;
int i;
if (!slot->in_use)
@@ -398,16 +401,10 @@ pg_get_replication_slots(PG_FUNCTION_ARGS)
break;
}
- if (max_slot_wal_keep_size_mb >= 0 &&
- (walstate == WALAVAIL_RESERVED || walstate == WALAVAIL_EXTENDED) &&
- ((last_removed_seg = XLogGetLastRemovedSegno()) != 0))
- {
- XLogRecPtr min_safe_lsn;
-
- XLogSegNoOffsetToRecPtr(last_removed_seg + 1, 0,
- wal_segment_size, min_safe_lsn);
- values[i++] = Int64GetDatum(min_safe_lsn);
- }
+ distance =
+ DistanceToWALHorizon(slot_contents.data.restart_lsn, currlsn);
+ if (distance > 0)
+ values[i++] = Int64GetDatum(distance);
else
nulls[i++] = true;
diff --git a/src/include/access/xlog.h b/src/include/access/xlog.h
index 77ac4e785f..1ec448c5d5 100644
--- a/src/include/access/xlog.h
+++ b/src/include/access/xlog.h
@@ -329,6 +329,7 @@ extern void InitXLOGAccess(void);
extern void CreateCheckPoint(int flags);
extern bool CreateRestartPoint(int flags);
extern WALAvailability GetWALAvailability(XLogRecPtr targetLSN);
+extern uint64 DistanceToWALHorizon(XLogRecPtr targetLSN, XLogRecPtr currptr);
extern XLogRecPtr CalculateMaxmumSafeLSN(void);
extern void XLogPutNextOid(Oid nextOid);
extern XLogRecPtr XLogRestorePoint(const char *rpName);
diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat
index 61f2c2f5b4..199fd994bd 100644
--- a/src/include/catalog/pg_proc.dat
+++ b/src/include/catalog/pg_proc.dat
@@ -10063,9 +10063,9 @@
proname => 'pg_get_replication_slots', prorows => '10', proisstrict => 'f',
proretset => 't', provolatile => 's', prorettype => 'record',
proargtypes => '',
- proallargtypes => '{name,name,text,oid,bool,bool,int4,xid,xid,pg_lsn,pg_lsn,text,pg_lsn}',
+ proallargtypes => '{name,name,text,oid,bool,bool,int4,xid,xid,pg_lsn,pg_lsn,text,int8}',
proargmodes => '{o,o,o,o,o,o,o,o,o,o,o,o,o}',
- proargnames => '{slot_name,plugin,slot_type,datoid,temporary,active,active_pid,xmin,catalog_xmin,restart_lsn,confirmed_flush_lsn,wal_status,min_safe_lsn}',
+ proargnames => '{slot_name,plugin,slot_type,datoid,temporary,active,active_pid,xmin,catalog_xmin,restart_lsn,confirmed_flush_lsn,wal_status,distance}',
prosrc => 'pg_get_replication_slots' },
{ oid => '3786', descr => 'set up a logical replication slot',
proname => 'pg_create_logical_replication_slot', provolatile => 'v',
diff --git a/src/test/recovery/t/019_replslot_limit.pl b/src/test/recovery/t/019_replslot_limit.pl
index 7d22ae5720..1c76d2d9e9 100644
--- a/src/test/recovery/t/019_replslot_limit.pl
+++ b/src/test/recovery/t/019_replslot_limit.pl
@@ -28,7 +28,7 @@ $node_master->safe_psql('postgres',
# The slot state and remain should be null before the first connection
my $result = $node_master->safe_psql('postgres',
- "SELECT restart_lsn IS NULL, wal_status is NULL, min_safe_lsn is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
+ "SELECT restart_lsn IS NULL, wal_status is NULL, distance is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
);
is($result, "t|t|t", 'check the state of non-reserved slot is "unknown"');
@@ -52,9 +52,9 @@ $node_master->wait_for_catchup($node_standby, 'replay', $start_lsn);
# Stop standby
$node_standby->stop;
-# Preparation done, the slot is the state "normal" now
+# Preparation done, the slot is the state "reserved" now
$result = $node_master->safe_psql('postgres',
- "SELECT wal_status, min_safe_lsn is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
+ "SELECT wal_status, distance is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
);
is($result, "reserved|t", 'check the catching-up state');
@@ -64,7 +64,7 @@ $node_master->safe_psql('postgres', "CHECKPOINT;");
# The slot is always "safe" when fitting max_wal_size
$result = $node_master->safe_psql('postgres',
- "SELECT wal_status, min_safe_lsn is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
+ "SELECT wal_status, distance is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
);
is($result, "reserved|t",
'check that it is safe if WAL fits in max_wal_size');
@@ -74,7 +74,7 @@ $node_master->safe_psql('postgres', "CHECKPOINT;");
# The slot is always "safe" when max_slot_wal_keep_size is not set
$result = $node_master->safe_psql('postgres',
- "SELECT wal_status, min_safe_lsn is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
+ "SELECT wal_status, distance is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
);
is($result, "reserved|t", 'check that slot is working');
@@ -94,9 +94,7 @@ max_slot_wal_keep_size = ${max_slot_wal_keep_size_mb}MB
));
$node_master->reload;
-# The slot is in safe state. The distance from the min_safe_lsn should
-# be as almost (max_slot_wal_keep_size - 1) times large as the segment
-# size
+# The slot is in safe state.
$result = $node_master->safe_psql('postgres',
"SELECT wal_status FROM pg_replication_slots WHERE slot_name = 'rep1'");
@@ -110,7 +108,7 @@ $node_master->safe_psql('postgres', "CHECKPOINT;");
$result = $node_master->safe_psql('postgres',
"SELECT wal_status FROM pg_replication_slots WHERE slot_name = 'rep1'");
is($result, "reserved",
- 'check that min_safe_lsn gets close to the current LSN');
+ 'check that distance gets close to the current LSN');
# The standby can reconnect to master
$node_standby->start;
@@ -154,7 +152,7 @@ advance_wal($node_master, 1);
# Slot gets into 'unreserved' state
$result = $node_master->safe_psql('postgres',
- "SELECT wal_status, min_safe_lsn is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
+ "SELECT wal_status, distance is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
);
is($result, "unreserved|t",
'check that the slot state changes to "unreserved"');
@@ -186,7 +184,7 @@ ok( find_in_log(
# This slot should be broken
$result = $node_master->safe_psql('postgres',
- "SELECT slot_name, active, restart_lsn IS NULL, wal_status, min_safe_lsn FROM pg_replication_slots WHERE slot_name = 'rep1'"
+ "SELECT slot_name, active, restart_lsn IS NULL, wal_status, distance FROM pg_replication_slots WHERE slot_name = 'rep1'"
);
is($result, "rep1|f|t|lost|",
'check that the slot became inactive and the state "lost" persists');
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b813e32215..392eab12dd 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1464,8 +1464,8 @@ pg_replication_slots| SELECT l.slot_name,
l.restart_lsn,
l.confirmed_flush_lsn,
l.wal_status,
- l.min_safe_lsn
- FROM (pg_get_replication_slots() l(slot_name, plugin, slot_type, datoid, temporary, active, active_pid, xmin, catalog_xmin, restart_lsn, confirmed_flush_lsn, wal_status, min_safe_lsn)
+ l.distance
+ FROM (pg_get_replication_slots() l(slot_name, plugin, slot_type, datoid, temporary, active, active_pid, xmin, catalog_xmin, restart_lsn, confirmed_flush_lsn, wal_status, distance)
LEFT JOIN pg_database d ON ((l.datoid = d.oid)));
pg_roles| SELECT pg_authid.rolname,
pg_authid.rolsuper,
--
2.18.4
----Next_Part(Wed_Jul__1_10_32_59_2020_161)----
^ permalink raw reply [nested|flat] 32+ messages in thread
* [PATCH v1] Replace min_safe_lsn with "distance" in pg_replication_slots
@ 2020-06-30 12:09 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 32+ messages in thread
From: Kyotaro Horiguchi @ 2020-06-30 12:09 UTC (permalink / raw)
pg_replication_slot.min_safe_lsn, which shows the oldest LSN kept in
pg_wal, is doubtful in usability for monitoring. Change it to
distance, which shows how many bytes the server can advance before the
slot loses required segments.
---
src/backend/access/transam/xlog.c | 39 +++++++++++++++++++++++
src/backend/catalog/system_views.sql | 2 +-
src/backend/replication/slotfuncs.c | 19 +++++------
src/include/access/xlog.h | 1 +
src/include/catalog/pg_proc.dat | 4 +--
src/test/recovery/t/019_replslot_limit.pl | 20 ++++++------
src/test/regress/expected/rules.out | 4 +--
7 files changed, 62 insertions(+), 27 deletions(-)
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index fd93bcfaeb..1f27639912 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -9570,6 +9570,45 @@ GetWALAvailability(XLogRecPtr targetLSN)
return WALAVAIL_REMOVED;
}
+/*
+ * Calculate how many bytes we can advance from currptr until the targetLSN is
+ * removed.
+ *
+ * Returns 0 if the distance is invalid.
+ */
+uint64
+DistanceToWALHorizon(XLogRecPtr targetLSN, XLogRecPtr currptr)
+{
+ XLogSegNo targetSeg;
+ XLogSegNo keepSegs;
+ XLogSegNo failSeg;
+ XLogRecPtr horizon;
+
+ XLByteToSeg(targetLSN, targetSeg, wal_segment_size);
+ keepSegs = 0;
+
+ /* no limit if max_slot_wal_keep_size is invalid */
+ if (max_slot_wal_keep_size_mb < 0)
+ return 0;
+
+ /* How many segments slots can keep? */
+ keepSegs = ConvertToXSegs(max_slot_wal_keep_size_mb, wal_segment_size);
+
+ /* override by wal_keep_segments if needed */
+ if (wal_keep_segments > keepSegs)
+ keepSegs = wal_keep_segments;
+
+ /* calculate the LSN where targetLSN is lost when currpos reaches */
+ failSeg = targetSeg + keepSegs + 1;
+ XLogSegNoOffsetToRecPtr(failSeg, 0, wal_segment_size, horizon);
+
+ /* If currptr already beyond the horizon, return zero. */
+ if (currptr > horizon)
+ return 0;
+
+ /* return the distance from currptr to the horizon */
+ return horizon - currptr;
+}
/*
* Retreat *logSegNo to the last segment that we need to retain because of
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index 5314e9348f..b9847a9f92 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -879,7 +879,7 @@ CREATE VIEW pg_replication_slots AS
L.restart_lsn,
L.confirmed_flush_lsn,
L.wal_status,
- L.min_safe_lsn
+ L.distance
FROM pg_get_replication_slots() AS L
LEFT JOIN pg_database D ON (L.datoid = D.oid);
diff --git a/src/backend/replication/slotfuncs.c b/src/backend/replication/slotfuncs.c
index 88033a79b2..532b3c5826 100644
--- a/src/backend/replication/slotfuncs.c
+++ b/src/backend/replication/slotfuncs.c
@@ -242,6 +242,7 @@ pg_get_replication_slots(PG_FUNCTION_ARGS)
Tuplestorestate *tupstore;
MemoryContext per_query_ctx;
MemoryContext oldcontext;
+ XLogRecPtr currlsn;
int slotno;
/* check to see if caller supports us returning a tuplestore */
@@ -274,6 +275,8 @@ pg_get_replication_slots(PG_FUNCTION_ARGS)
MemoryContextSwitchTo(oldcontext);
+ currlsn = GetXLogWriteRecPtr();
+
LWLockAcquire(ReplicationSlotControlLock, LW_SHARED);
for (slotno = 0; slotno < max_replication_slots; slotno++)
{
@@ -282,7 +285,7 @@ pg_get_replication_slots(PG_FUNCTION_ARGS)
Datum values[PG_GET_REPLICATION_SLOTS_COLS];
bool nulls[PG_GET_REPLICATION_SLOTS_COLS];
WALAvailability walstate;
- XLogSegNo last_removed_seg;
+ uint32 distance;
int i;
if (!slot->in_use)
@@ -398,16 +401,10 @@ pg_get_replication_slots(PG_FUNCTION_ARGS)
break;
}
- if (max_slot_wal_keep_size_mb >= 0 &&
- (walstate == WALAVAIL_RESERVED || walstate == WALAVAIL_EXTENDED) &&
- ((last_removed_seg = XLogGetLastRemovedSegno()) != 0))
- {
- XLogRecPtr min_safe_lsn;
-
- XLogSegNoOffsetToRecPtr(last_removed_seg + 1, 0,
- wal_segment_size, min_safe_lsn);
- values[i++] = Int64GetDatum(min_safe_lsn);
- }
+ distance =
+ DistanceToWALHorizon(slot_contents.data.restart_lsn, currlsn);
+ if (distance > 0)
+ values[i++] = Int64GetDatum(distance);
else
nulls[i++] = true;
diff --git a/src/include/access/xlog.h b/src/include/access/xlog.h
index 77ac4e785f..1ec448c5d5 100644
--- a/src/include/access/xlog.h
+++ b/src/include/access/xlog.h
@@ -329,6 +329,7 @@ extern void InitXLOGAccess(void);
extern void CreateCheckPoint(int flags);
extern bool CreateRestartPoint(int flags);
extern WALAvailability GetWALAvailability(XLogRecPtr targetLSN);
+extern uint64 DistanceToWALHorizon(XLogRecPtr targetLSN, XLogRecPtr currptr);
extern XLogRecPtr CalculateMaxmumSafeLSN(void);
extern void XLogPutNextOid(Oid nextOid);
extern XLogRecPtr XLogRestorePoint(const char *rpName);
diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat
index 61f2c2f5b4..199fd994bd 100644
--- a/src/include/catalog/pg_proc.dat
+++ b/src/include/catalog/pg_proc.dat
@@ -10063,9 +10063,9 @@
proname => 'pg_get_replication_slots', prorows => '10', proisstrict => 'f',
proretset => 't', provolatile => 's', prorettype => 'record',
proargtypes => '',
- proallargtypes => '{name,name,text,oid,bool,bool,int4,xid,xid,pg_lsn,pg_lsn,text,pg_lsn}',
+ proallargtypes => '{name,name,text,oid,bool,bool,int4,xid,xid,pg_lsn,pg_lsn,text,int8}',
proargmodes => '{o,o,o,o,o,o,o,o,o,o,o,o,o}',
- proargnames => '{slot_name,plugin,slot_type,datoid,temporary,active,active_pid,xmin,catalog_xmin,restart_lsn,confirmed_flush_lsn,wal_status,min_safe_lsn}',
+ proargnames => '{slot_name,plugin,slot_type,datoid,temporary,active,active_pid,xmin,catalog_xmin,restart_lsn,confirmed_flush_lsn,wal_status,distance}',
prosrc => 'pg_get_replication_slots' },
{ oid => '3786', descr => 'set up a logical replication slot',
proname => 'pg_create_logical_replication_slot', provolatile => 'v',
diff --git a/src/test/recovery/t/019_replslot_limit.pl b/src/test/recovery/t/019_replslot_limit.pl
index 7d22ae5720..1c76d2d9e9 100644
--- a/src/test/recovery/t/019_replslot_limit.pl
+++ b/src/test/recovery/t/019_replslot_limit.pl
@@ -28,7 +28,7 @@ $node_master->safe_psql('postgres',
# The slot state and remain should be null before the first connection
my $result = $node_master->safe_psql('postgres',
- "SELECT restart_lsn IS NULL, wal_status is NULL, min_safe_lsn is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
+ "SELECT restart_lsn IS NULL, wal_status is NULL, distance is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
);
is($result, "t|t|t", 'check the state of non-reserved slot is "unknown"');
@@ -52,9 +52,9 @@ $node_master->wait_for_catchup($node_standby, 'replay', $start_lsn);
# Stop standby
$node_standby->stop;
-# Preparation done, the slot is the state "normal" now
+# Preparation done, the slot is the state "reserved" now
$result = $node_master->safe_psql('postgres',
- "SELECT wal_status, min_safe_lsn is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
+ "SELECT wal_status, distance is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
);
is($result, "reserved|t", 'check the catching-up state');
@@ -64,7 +64,7 @@ $node_master->safe_psql('postgres', "CHECKPOINT;");
# The slot is always "safe" when fitting max_wal_size
$result = $node_master->safe_psql('postgres',
- "SELECT wal_status, min_safe_lsn is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
+ "SELECT wal_status, distance is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
);
is($result, "reserved|t",
'check that it is safe if WAL fits in max_wal_size');
@@ -74,7 +74,7 @@ $node_master->safe_psql('postgres', "CHECKPOINT;");
# The slot is always "safe" when max_slot_wal_keep_size is not set
$result = $node_master->safe_psql('postgres',
- "SELECT wal_status, min_safe_lsn is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
+ "SELECT wal_status, distance is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
);
is($result, "reserved|t", 'check that slot is working');
@@ -94,9 +94,7 @@ max_slot_wal_keep_size = ${max_slot_wal_keep_size_mb}MB
));
$node_master->reload;
-# The slot is in safe state. The distance from the min_safe_lsn should
-# be as almost (max_slot_wal_keep_size - 1) times large as the segment
-# size
+# The slot is in safe state.
$result = $node_master->safe_psql('postgres',
"SELECT wal_status FROM pg_replication_slots WHERE slot_name = 'rep1'");
@@ -110,7 +108,7 @@ $node_master->safe_psql('postgres', "CHECKPOINT;");
$result = $node_master->safe_psql('postgres',
"SELECT wal_status FROM pg_replication_slots WHERE slot_name = 'rep1'");
is($result, "reserved",
- 'check that min_safe_lsn gets close to the current LSN');
+ 'check that distance gets close to the current LSN');
# The standby can reconnect to master
$node_standby->start;
@@ -154,7 +152,7 @@ advance_wal($node_master, 1);
# Slot gets into 'unreserved' state
$result = $node_master->safe_psql('postgres',
- "SELECT wal_status, min_safe_lsn is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
+ "SELECT wal_status, distance is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
);
is($result, "unreserved|t",
'check that the slot state changes to "unreserved"');
@@ -186,7 +184,7 @@ ok( find_in_log(
# This slot should be broken
$result = $node_master->safe_psql('postgres',
- "SELECT slot_name, active, restart_lsn IS NULL, wal_status, min_safe_lsn FROM pg_replication_slots WHERE slot_name = 'rep1'"
+ "SELECT slot_name, active, restart_lsn IS NULL, wal_status, distance FROM pg_replication_slots WHERE slot_name = 'rep1'"
);
is($result, "rep1|f|t|lost|",
'check that the slot became inactive and the state "lost" persists');
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b813e32215..392eab12dd 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1464,8 +1464,8 @@ pg_replication_slots| SELECT l.slot_name,
l.restart_lsn,
l.confirmed_flush_lsn,
l.wal_status,
- l.min_safe_lsn
- FROM (pg_get_replication_slots() l(slot_name, plugin, slot_type, datoid, temporary, active, active_pid, xmin, catalog_xmin, restart_lsn, confirmed_flush_lsn, wal_status, min_safe_lsn)
+ l.distance
+ FROM (pg_get_replication_slots() l(slot_name, plugin, slot_type, datoid, temporary, active, active_pid, xmin, catalog_xmin, restart_lsn, confirmed_flush_lsn, wal_status, distance)
LEFT JOIN pg_database d ON ((l.datoid = d.oid)));
pg_roles| SELECT pg_authid.rolname,
pg_authid.rolsuper,
--
2.18.4
----Next_Part(Wed_Jul__1_10_32_59_2020_161)----
^ permalink raw reply [nested|flat] 32+ messages in thread
* [PATCH v1] Replace min_safe_lsn with "distance" in pg_replication_slots
@ 2020-06-30 12:09 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 32+ messages in thread
From: Kyotaro Horiguchi @ 2020-06-30 12:09 UTC (permalink / raw)
pg_replication_slot.min_safe_lsn, which shows the oldest LSN kept in
pg_wal, is doubtful in usability for monitoring. Change it to
distance, which shows how many bytes the server can advance before the
slot loses required segments.
---
src/backend/access/transam/xlog.c | 39 +++++++++++++++++++++++
src/backend/catalog/system_views.sql | 2 +-
src/backend/replication/slotfuncs.c | 19 +++++------
src/include/access/xlog.h | 1 +
src/include/catalog/pg_proc.dat | 4 +--
src/test/recovery/t/019_replslot_limit.pl | 20 ++++++------
src/test/regress/expected/rules.out | 4 +--
7 files changed, 62 insertions(+), 27 deletions(-)
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index fd93bcfaeb..1f27639912 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -9570,6 +9570,45 @@ GetWALAvailability(XLogRecPtr targetLSN)
return WALAVAIL_REMOVED;
}
+/*
+ * Calculate how many bytes we can advance from currptr until the targetLSN is
+ * removed.
+ *
+ * Returns 0 if the distance is invalid.
+ */
+uint64
+DistanceToWALHorizon(XLogRecPtr targetLSN, XLogRecPtr currptr)
+{
+ XLogSegNo targetSeg;
+ XLogSegNo keepSegs;
+ XLogSegNo failSeg;
+ XLogRecPtr horizon;
+
+ XLByteToSeg(targetLSN, targetSeg, wal_segment_size);
+ keepSegs = 0;
+
+ /* no limit if max_slot_wal_keep_size is invalid */
+ if (max_slot_wal_keep_size_mb < 0)
+ return 0;
+
+ /* How many segments slots can keep? */
+ keepSegs = ConvertToXSegs(max_slot_wal_keep_size_mb, wal_segment_size);
+
+ /* override by wal_keep_segments if needed */
+ if (wal_keep_segments > keepSegs)
+ keepSegs = wal_keep_segments;
+
+ /* calculate the LSN where targetLSN is lost when currpos reaches */
+ failSeg = targetSeg + keepSegs + 1;
+ XLogSegNoOffsetToRecPtr(failSeg, 0, wal_segment_size, horizon);
+
+ /* If currptr already beyond the horizon, return zero. */
+ if (currptr > horizon)
+ return 0;
+
+ /* return the distance from currptr to the horizon */
+ return horizon - currptr;
+}
/*
* Retreat *logSegNo to the last segment that we need to retain because of
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index 5314e9348f..b9847a9f92 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -879,7 +879,7 @@ CREATE VIEW pg_replication_slots AS
L.restart_lsn,
L.confirmed_flush_lsn,
L.wal_status,
- L.min_safe_lsn
+ L.distance
FROM pg_get_replication_slots() AS L
LEFT JOIN pg_database D ON (L.datoid = D.oid);
diff --git a/src/backend/replication/slotfuncs.c b/src/backend/replication/slotfuncs.c
index 88033a79b2..532b3c5826 100644
--- a/src/backend/replication/slotfuncs.c
+++ b/src/backend/replication/slotfuncs.c
@@ -242,6 +242,7 @@ pg_get_replication_slots(PG_FUNCTION_ARGS)
Tuplestorestate *tupstore;
MemoryContext per_query_ctx;
MemoryContext oldcontext;
+ XLogRecPtr currlsn;
int slotno;
/* check to see if caller supports us returning a tuplestore */
@@ -274,6 +275,8 @@ pg_get_replication_slots(PG_FUNCTION_ARGS)
MemoryContextSwitchTo(oldcontext);
+ currlsn = GetXLogWriteRecPtr();
+
LWLockAcquire(ReplicationSlotControlLock, LW_SHARED);
for (slotno = 0; slotno < max_replication_slots; slotno++)
{
@@ -282,7 +285,7 @@ pg_get_replication_slots(PG_FUNCTION_ARGS)
Datum values[PG_GET_REPLICATION_SLOTS_COLS];
bool nulls[PG_GET_REPLICATION_SLOTS_COLS];
WALAvailability walstate;
- XLogSegNo last_removed_seg;
+ uint32 distance;
int i;
if (!slot->in_use)
@@ -398,16 +401,10 @@ pg_get_replication_slots(PG_FUNCTION_ARGS)
break;
}
- if (max_slot_wal_keep_size_mb >= 0 &&
- (walstate == WALAVAIL_RESERVED || walstate == WALAVAIL_EXTENDED) &&
- ((last_removed_seg = XLogGetLastRemovedSegno()) != 0))
- {
- XLogRecPtr min_safe_lsn;
-
- XLogSegNoOffsetToRecPtr(last_removed_seg + 1, 0,
- wal_segment_size, min_safe_lsn);
- values[i++] = Int64GetDatum(min_safe_lsn);
- }
+ distance =
+ DistanceToWALHorizon(slot_contents.data.restart_lsn, currlsn);
+ if (distance > 0)
+ values[i++] = Int64GetDatum(distance);
else
nulls[i++] = true;
diff --git a/src/include/access/xlog.h b/src/include/access/xlog.h
index 77ac4e785f..1ec448c5d5 100644
--- a/src/include/access/xlog.h
+++ b/src/include/access/xlog.h
@@ -329,6 +329,7 @@ extern void InitXLOGAccess(void);
extern void CreateCheckPoint(int flags);
extern bool CreateRestartPoint(int flags);
extern WALAvailability GetWALAvailability(XLogRecPtr targetLSN);
+extern uint64 DistanceToWALHorizon(XLogRecPtr targetLSN, XLogRecPtr currptr);
extern XLogRecPtr CalculateMaxmumSafeLSN(void);
extern void XLogPutNextOid(Oid nextOid);
extern XLogRecPtr XLogRestorePoint(const char *rpName);
diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat
index 61f2c2f5b4..199fd994bd 100644
--- a/src/include/catalog/pg_proc.dat
+++ b/src/include/catalog/pg_proc.dat
@@ -10063,9 +10063,9 @@
proname => 'pg_get_replication_slots', prorows => '10', proisstrict => 'f',
proretset => 't', provolatile => 's', prorettype => 'record',
proargtypes => '',
- proallargtypes => '{name,name,text,oid,bool,bool,int4,xid,xid,pg_lsn,pg_lsn,text,pg_lsn}',
+ proallargtypes => '{name,name,text,oid,bool,bool,int4,xid,xid,pg_lsn,pg_lsn,text,int8}',
proargmodes => '{o,o,o,o,o,o,o,o,o,o,o,o,o}',
- proargnames => '{slot_name,plugin,slot_type,datoid,temporary,active,active_pid,xmin,catalog_xmin,restart_lsn,confirmed_flush_lsn,wal_status,min_safe_lsn}',
+ proargnames => '{slot_name,plugin,slot_type,datoid,temporary,active,active_pid,xmin,catalog_xmin,restart_lsn,confirmed_flush_lsn,wal_status,distance}',
prosrc => 'pg_get_replication_slots' },
{ oid => '3786', descr => 'set up a logical replication slot',
proname => 'pg_create_logical_replication_slot', provolatile => 'v',
diff --git a/src/test/recovery/t/019_replslot_limit.pl b/src/test/recovery/t/019_replslot_limit.pl
index 7d22ae5720..1c76d2d9e9 100644
--- a/src/test/recovery/t/019_replslot_limit.pl
+++ b/src/test/recovery/t/019_replslot_limit.pl
@@ -28,7 +28,7 @@ $node_master->safe_psql('postgres',
# The slot state and remain should be null before the first connection
my $result = $node_master->safe_psql('postgres',
- "SELECT restart_lsn IS NULL, wal_status is NULL, min_safe_lsn is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
+ "SELECT restart_lsn IS NULL, wal_status is NULL, distance is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
);
is($result, "t|t|t", 'check the state of non-reserved slot is "unknown"');
@@ -52,9 +52,9 @@ $node_master->wait_for_catchup($node_standby, 'replay', $start_lsn);
# Stop standby
$node_standby->stop;
-# Preparation done, the slot is the state "normal" now
+# Preparation done, the slot is the state "reserved" now
$result = $node_master->safe_psql('postgres',
- "SELECT wal_status, min_safe_lsn is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
+ "SELECT wal_status, distance is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
);
is($result, "reserved|t", 'check the catching-up state');
@@ -64,7 +64,7 @@ $node_master->safe_psql('postgres', "CHECKPOINT;");
# The slot is always "safe" when fitting max_wal_size
$result = $node_master->safe_psql('postgres',
- "SELECT wal_status, min_safe_lsn is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
+ "SELECT wal_status, distance is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
);
is($result, "reserved|t",
'check that it is safe if WAL fits in max_wal_size');
@@ -74,7 +74,7 @@ $node_master->safe_psql('postgres', "CHECKPOINT;");
# The slot is always "safe" when max_slot_wal_keep_size is not set
$result = $node_master->safe_psql('postgres',
- "SELECT wal_status, min_safe_lsn is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
+ "SELECT wal_status, distance is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
);
is($result, "reserved|t", 'check that slot is working');
@@ -94,9 +94,7 @@ max_slot_wal_keep_size = ${max_slot_wal_keep_size_mb}MB
));
$node_master->reload;
-# The slot is in safe state. The distance from the min_safe_lsn should
-# be as almost (max_slot_wal_keep_size - 1) times large as the segment
-# size
+# The slot is in safe state.
$result = $node_master->safe_psql('postgres',
"SELECT wal_status FROM pg_replication_slots WHERE slot_name = 'rep1'");
@@ -110,7 +108,7 @@ $node_master->safe_psql('postgres', "CHECKPOINT;");
$result = $node_master->safe_psql('postgres',
"SELECT wal_status FROM pg_replication_slots WHERE slot_name = 'rep1'");
is($result, "reserved",
- 'check that min_safe_lsn gets close to the current LSN');
+ 'check that distance gets close to the current LSN');
# The standby can reconnect to master
$node_standby->start;
@@ -154,7 +152,7 @@ advance_wal($node_master, 1);
# Slot gets into 'unreserved' state
$result = $node_master->safe_psql('postgres',
- "SELECT wal_status, min_safe_lsn is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
+ "SELECT wal_status, distance is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
);
is($result, "unreserved|t",
'check that the slot state changes to "unreserved"');
@@ -186,7 +184,7 @@ ok( find_in_log(
# This slot should be broken
$result = $node_master->safe_psql('postgres',
- "SELECT slot_name, active, restart_lsn IS NULL, wal_status, min_safe_lsn FROM pg_replication_slots WHERE slot_name = 'rep1'"
+ "SELECT slot_name, active, restart_lsn IS NULL, wal_status, distance FROM pg_replication_slots WHERE slot_name = 'rep1'"
);
is($result, "rep1|f|t|lost|",
'check that the slot became inactive and the state "lost" persists');
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b813e32215..392eab12dd 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1464,8 +1464,8 @@ pg_replication_slots| SELECT l.slot_name,
l.restart_lsn,
l.confirmed_flush_lsn,
l.wal_status,
- l.min_safe_lsn
- FROM (pg_get_replication_slots() l(slot_name, plugin, slot_type, datoid, temporary, active, active_pid, xmin, catalog_xmin, restart_lsn, confirmed_flush_lsn, wal_status, min_safe_lsn)
+ l.distance
+ FROM (pg_get_replication_slots() l(slot_name, plugin, slot_type, datoid, temporary, active, active_pid, xmin, catalog_xmin, restart_lsn, confirmed_flush_lsn, wal_status, distance)
LEFT JOIN pg_database d ON ((l.datoid = d.oid)));
pg_roles| SELECT pg_authid.rolname,
pg_authid.rolsuper,
--
2.18.4
----Next_Part(Wed_Jul__1_10_32_59_2020_161)----
^ permalink raw reply [nested|flat] 32+ messages in thread
* [PATCH v1] Replace min_safe_lsn with "distance" in pg_replication_slots
@ 2020-06-30 12:09 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 32+ messages in thread
From: Kyotaro Horiguchi @ 2020-06-30 12:09 UTC (permalink / raw)
pg_replication_slot.min_safe_lsn, which shows the oldest LSN kept in
pg_wal, is doubtful in usability for monitoring. Change it to
distance, which shows how many bytes the server can advance before the
slot loses required segments.
---
src/backend/access/transam/xlog.c | 39 +++++++++++++++++++++++
src/backend/catalog/system_views.sql | 2 +-
src/backend/replication/slotfuncs.c | 19 +++++------
src/include/access/xlog.h | 1 +
src/include/catalog/pg_proc.dat | 4 +--
src/test/recovery/t/019_replslot_limit.pl | 20 ++++++------
src/test/regress/expected/rules.out | 4 +--
7 files changed, 62 insertions(+), 27 deletions(-)
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index fd93bcfaeb..1f27639912 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -9570,6 +9570,45 @@ GetWALAvailability(XLogRecPtr targetLSN)
return WALAVAIL_REMOVED;
}
+/*
+ * Calculate how many bytes we can advance from currptr until the targetLSN is
+ * removed.
+ *
+ * Returns 0 if the distance is invalid.
+ */
+uint64
+DistanceToWALHorizon(XLogRecPtr targetLSN, XLogRecPtr currptr)
+{
+ XLogSegNo targetSeg;
+ XLogSegNo keepSegs;
+ XLogSegNo failSeg;
+ XLogRecPtr horizon;
+
+ XLByteToSeg(targetLSN, targetSeg, wal_segment_size);
+ keepSegs = 0;
+
+ /* no limit if max_slot_wal_keep_size is invalid */
+ if (max_slot_wal_keep_size_mb < 0)
+ return 0;
+
+ /* How many segments slots can keep? */
+ keepSegs = ConvertToXSegs(max_slot_wal_keep_size_mb, wal_segment_size);
+
+ /* override by wal_keep_segments if needed */
+ if (wal_keep_segments > keepSegs)
+ keepSegs = wal_keep_segments;
+
+ /* calculate the LSN where targetLSN is lost when currpos reaches */
+ failSeg = targetSeg + keepSegs + 1;
+ XLogSegNoOffsetToRecPtr(failSeg, 0, wal_segment_size, horizon);
+
+ /* If currptr already beyond the horizon, return zero. */
+ if (currptr > horizon)
+ return 0;
+
+ /* return the distance from currptr to the horizon */
+ return horizon - currptr;
+}
/*
* Retreat *logSegNo to the last segment that we need to retain because of
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index 5314e9348f..b9847a9f92 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -879,7 +879,7 @@ CREATE VIEW pg_replication_slots AS
L.restart_lsn,
L.confirmed_flush_lsn,
L.wal_status,
- L.min_safe_lsn
+ L.distance
FROM pg_get_replication_slots() AS L
LEFT JOIN pg_database D ON (L.datoid = D.oid);
diff --git a/src/backend/replication/slotfuncs.c b/src/backend/replication/slotfuncs.c
index 88033a79b2..532b3c5826 100644
--- a/src/backend/replication/slotfuncs.c
+++ b/src/backend/replication/slotfuncs.c
@@ -242,6 +242,7 @@ pg_get_replication_slots(PG_FUNCTION_ARGS)
Tuplestorestate *tupstore;
MemoryContext per_query_ctx;
MemoryContext oldcontext;
+ XLogRecPtr currlsn;
int slotno;
/* check to see if caller supports us returning a tuplestore */
@@ -274,6 +275,8 @@ pg_get_replication_slots(PG_FUNCTION_ARGS)
MemoryContextSwitchTo(oldcontext);
+ currlsn = GetXLogWriteRecPtr();
+
LWLockAcquire(ReplicationSlotControlLock, LW_SHARED);
for (slotno = 0; slotno < max_replication_slots; slotno++)
{
@@ -282,7 +285,7 @@ pg_get_replication_slots(PG_FUNCTION_ARGS)
Datum values[PG_GET_REPLICATION_SLOTS_COLS];
bool nulls[PG_GET_REPLICATION_SLOTS_COLS];
WALAvailability walstate;
- XLogSegNo last_removed_seg;
+ uint32 distance;
int i;
if (!slot->in_use)
@@ -398,16 +401,10 @@ pg_get_replication_slots(PG_FUNCTION_ARGS)
break;
}
- if (max_slot_wal_keep_size_mb >= 0 &&
- (walstate == WALAVAIL_RESERVED || walstate == WALAVAIL_EXTENDED) &&
- ((last_removed_seg = XLogGetLastRemovedSegno()) != 0))
- {
- XLogRecPtr min_safe_lsn;
-
- XLogSegNoOffsetToRecPtr(last_removed_seg + 1, 0,
- wal_segment_size, min_safe_lsn);
- values[i++] = Int64GetDatum(min_safe_lsn);
- }
+ distance =
+ DistanceToWALHorizon(slot_contents.data.restart_lsn, currlsn);
+ if (distance > 0)
+ values[i++] = Int64GetDatum(distance);
else
nulls[i++] = true;
diff --git a/src/include/access/xlog.h b/src/include/access/xlog.h
index 77ac4e785f..1ec448c5d5 100644
--- a/src/include/access/xlog.h
+++ b/src/include/access/xlog.h
@@ -329,6 +329,7 @@ extern void InitXLOGAccess(void);
extern void CreateCheckPoint(int flags);
extern bool CreateRestartPoint(int flags);
extern WALAvailability GetWALAvailability(XLogRecPtr targetLSN);
+extern uint64 DistanceToWALHorizon(XLogRecPtr targetLSN, XLogRecPtr currptr);
extern XLogRecPtr CalculateMaxmumSafeLSN(void);
extern void XLogPutNextOid(Oid nextOid);
extern XLogRecPtr XLogRestorePoint(const char *rpName);
diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat
index 61f2c2f5b4..199fd994bd 100644
--- a/src/include/catalog/pg_proc.dat
+++ b/src/include/catalog/pg_proc.dat
@@ -10063,9 +10063,9 @@
proname => 'pg_get_replication_slots', prorows => '10', proisstrict => 'f',
proretset => 't', provolatile => 's', prorettype => 'record',
proargtypes => '',
- proallargtypes => '{name,name,text,oid,bool,bool,int4,xid,xid,pg_lsn,pg_lsn,text,pg_lsn}',
+ proallargtypes => '{name,name,text,oid,bool,bool,int4,xid,xid,pg_lsn,pg_lsn,text,int8}',
proargmodes => '{o,o,o,o,o,o,o,o,o,o,o,o,o}',
- proargnames => '{slot_name,plugin,slot_type,datoid,temporary,active,active_pid,xmin,catalog_xmin,restart_lsn,confirmed_flush_lsn,wal_status,min_safe_lsn}',
+ proargnames => '{slot_name,plugin,slot_type,datoid,temporary,active,active_pid,xmin,catalog_xmin,restart_lsn,confirmed_flush_lsn,wal_status,distance}',
prosrc => 'pg_get_replication_slots' },
{ oid => '3786', descr => 'set up a logical replication slot',
proname => 'pg_create_logical_replication_slot', provolatile => 'v',
diff --git a/src/test/recovery/t/019_replslot_limit.pl b/src/test/recovery/t/019_replslot_limit.pl
index 7d22ae5720..1c76d2d9e9 100644
--- a/src/test/recovery/t/019_replslot_limit.pl
+++ b/src/test/recovery/t/019_replslot_limit.pl
@@ -28,7 +28,7 @@ $node_master->safe_psql('postgres',
# The slot state and remain should be null before the first connection
my $result = $node_master->safe_psql('postgres',
- "SELECT restart_lsn IS NULL, wal_status is NULL, min_safe_lsn is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
+ "SELECT restart_lsn IS NULL, wal_status is NULL, distance is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
);
is($result, "t|t|t", 'check the state of non-reserved slot is "unknown"');
@@ -52,9 +52,9 @@ $node_master->wait_for_catchup($node_standby, 'replay', $start_lsn);
# Stop standby
$node_standby->stop;
-# Preparation done, the slot is the state "normal" now
+# Preparation done, the slot is the state "reserved" now
$result = $node_master->safe_psql('postgres',
- "SELECT wal_status, min_safe_lsn is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
+ "SELECT wal_status, distance is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
);
is($result, "reserved|t", 'check the catching-up state');
@@ -64,7 +64,7 @@ $node_master->safe_psql('postgres', "CHECKPOINT;");
# The slot is always "safe" when fitting max_wal_size
$result = $node_master->safe_psql('postgres',
- "SELECT wal_status, min_safe_lsn is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
+ "SELECT wal_status, distance is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
);
is($result, "reserved|t",
'check that it is safe if WAL fits in max_wal_size');
@@ -74,7 +74,7 @@ $node_master->safe_psql('postgres', "CHECKPOINT;");
# The slot is always "safe" when max_slot_wal_keep_size is not set
$result = $node_master->safe_psql('postgres',
- "SELECT wal_status, min_safe_lsn is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
+ "SELECT wal_status, distance is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
);
is($result, "reserved|t", 'check that slot is working');
@@ -94,9 +94,7 @@ max_slot_wal_keep_size = ${max_slot_wal_keep_size_mb}MB
));
$node_master->reload;
-# The slot is in safe state. The distance from the min_safe_lsn should
-# be as almost (max_slot_wal_keep_size - 1) times large as the segment
-# size
+# The slot is in safe state.
$result = $node_master->safe_psql('postgres',
"SELECT wal_status FROM pg_replication_slots WHERE slot_name = 'rep1'");
@@ -110,7 +108,7 @@ $node_master->safe_psql('postgres', "CHECKPOINT;");
$result = $node_master->safe_psql('postgres',
"SELECT wal_status FROM pg_replication_slots WHERE slot_name = 'rep1'");
is($result, "reserved",
- 'check that min_safe_lsn gets close to the current LSN');
+ 'check that distance gets close to the current LSN');
# The standby can reconnect to master
$node_standby->start;
@@ -154,7 +152,7 @@ advance_wal($node_master, 1);
# Slot gets into 'unreserved' state
$result = $node_master->safe_psql('postgres',
- "SELECT wal_status, min_safe_lsn is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
+ "SELECT wal_status, distance is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
);
is($result, "unreserved|t",
'check that the slot state changes to "unreserved"');
@@ -186,7 +184,7 @@ ok( find_in_log(
# This slot should be broken
$result = $node_master->safe_psql('postgres',
- "SELECT slot_name, active, restart_lsn IS NULL, wal_status, min_safe_lsn FROM pg_replication_slots WHERE slot_name = 'rep1'"
+ "SELECT slot_name, active, restart_lsn IS NULL, wal_status, distance FROM pg_replication_slots WHERE slot_name = 'rep1'"
);
is($result, "rep1|f|t|lost|",
'check that the slot became inactive and the state "lost" persists');
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b813e32215..392eab12dd 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1464,8 +1464,8 @@ pg_replication_slots| SELECT l.slot_name,
l.restart_lsn,
l.confirmed_flush_lsn,
l.wal_status,
- l.min_safe_lsn
- FROM (pg_get_replication_slots() l(slot_name, plugin, slot_type, datoid, temporary, active, active_pid, xmin, catalog_xmin, restart_lsn, confirmed_flush_lsn, wal_status, min_safe_lsn)
+ l.distance
+ FROM (pg_get_replication_slots() l(slot_name, plugin, slot_type, datoid, temporary, active, active_pid, xmin, catalog_xmin, restart_lsn, confirmed_flush_lsn, wal_status, distance)
LEFT JOIN pg_database d ON ((l.datoid = d.oid)));
pg_roles| SELECT pg_authid.rolname,
pg_authid.rolsuper,
--
2.18.4
----Next_Part(Wed_Jul__1_10_32_59_2020_161)----
^ permalink raw reply [nested|flat] 32+ messages in thread
* [PATCH v1] Replace min_safe_lsn with "distance" in pg_replication_slots
@ 2020-06-30 12:09 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 32+ messages in thread
From: Kyotaro Horiguchi @ 2020-06-30 12:09 UTC (permalink / raw)
pg_replication_slot.min_safe_lsn, which shows the oldest LSN kept in
pg_wal, is doubtful in usability for monitoring. Change it to
distance, which shows how many bytes the server can advance before the
slot loses required segments.
---
src/backend/access/transam/xlog.c | 39 +++++++++++++++++++++++
src/backend/catalog/system_views.sql | 2 +-
src/backend/replication/slotfuncs.c | 19 +++++------
src/include/access/xlog.h | 1 +
src/include/catalog/pg_proc.dat | 4 +--
src/test/recovery/t/019_replslot_limit.pl | 20 ++++++------
src/test/regress/expected/rules.out | 4 +--
7 files changed, 62 insertions(+), 27 deletions(-)
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index fd93bcfaeb..1f27639912 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -9570,6 +9570,45 @@ GetWALAvailability(XLogRecPtr targetLSN)
return WALAVAIL_REMOVED;
}
+/*
+ * Calculate how many bytes we can advance from currptr until the targetLSN is
+ * removed.
+ *
+ * Returns 0 if the distance is invalid.
+ */
+uint64
+DistanceToWALHorizon(XLogRecPtr targetLSN, XLogRecPtr currptr)
+{
+ XLogSegNo targetSeg;
+ XLogSegNo keepSegs;
+ XLogSegNo failSeg;
+ XLogRecPtr horizon;
+
+ XLByteToSeg(targetLSN, targetSeg, wal_segment_size);
+ keepSegs = 0;
+
+ /* no limit if max_slot_wal_keep_size is invalid */
+ if (max_slot_wal_keep_size_mb < 0)
+ return 0;
+
+ /* How many segments slots can keep? */
+ keepSegs = ConvertToXSegs(max_slot_wal_keep_size_mb, wal_segment_size);
+
+ /* override by wal_keep_segments if needed */
+ if (wal_keep_segments > keepSegs)
+ keepSegs = wal_keep_segments;
+
+ /* calculate the LSN where targetLSN is lost when currpos reaches */
+ failSeg = targetSeg + keepSegs + 1;
+ XLogSegNoOffsetToRecPtr(failSeg, 0, wal_segment_size, horizon);
+
+ /* If currptr already beyond the horizon, return zero. */
+ if (currptr > horizon)
+ return 0;
+
+ /* return the distance from currptr to the horizon */
+ return horizon - currptr;
+}
/*
* Retreat *logSegNo to the last segment that we need to retain because of
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index 5314e9348f..b9847a9f92 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -879,7 +879,7 @@ CREATE VIEW pg_replication_slots AS
L.restart_lsn,
L.confirmed_flush_lsn,
L.wal_status,
- L.min_safe_lsn
+ L.distance
FROM pg_get_replication_slots() AS L
LEFT JOIN pg_database D ON (L.datoid = D.oid);
diff --git a/src/backend/replication/slotfuncs.c b/src/backend/replication/slotfuncs.c
index 88033a79b2..532b3c5826 100644
--- a/src/backend/replication/slotfuncs.c
+++ b/src/backend/replication/slotfuncs.c
@@ -242,6 +242,7 @@ pg_get_replication_slots(PG_FUNCTION_ARGS)
Tuplestorestate *tupstore;
MemoryContext per_query_ctx;
MemoryContext oldcontext;
+ XLogRecPtr currlsn;
int slotno;
/* check to see if caller supports us returning a tuplestore */
@@ -274,6 +275,8 @@ pg_get_replication_slots(PG_FUNCTION_ARGS)
MemoryContextSwitchTo(oldcontext);
+ currlsn = GetXLogWriteRecPtr();
+
LWLockAcquire(ReplicationSlotControlLock, LW_SHARED);
for (slotno = 0; slotno < max_replication_slots; slotno++)
{
@@ -282,7 +285,7 @@ pg_get_replication_slots(PG_FUNCTION_ARGS)
Datum values[PG_GET_REPLICATION_SLOTS_COLS];
bool nulls[PG_GET_REPLICATION_SLOTS_COLS];
WALAvailability walstate;
- XLogSegNo last_removed_seg;
+ uint32 distance;
int i;
if (!slot->in_use)
@@ -398,16 +401,10 @@ pg_get_replication_slots(PG_FUNCTION_ARGS)
break;
}
- if (max_slot_wal_keep_size_mb >= 0 &&
- (walstate == WALAVAIL_RESERVED || walstate == WALAVAIL_EXTENDED) &&
- ((last_removed_seg = XLogGetLastRemovedSegno()) != 0))
- {
- XLogRecPtr min_safe_lsn;
-
- XLogSegNoOffsetToRecPtr(last_removed_seg + 1, 0,
- wal_segment_size, min_safe_lsn);
- values[i++] = Int64GetDatum(min_safe_lsn);
- }
+ distance =
+ DistanceToWALHorizon(slot_contents.data.restart_lsn, currlsn);
+ if (distance > 0)
+ values[i++] = Int64GetDatum(distance);
else
nulls[i++] = true;
diff --git a/src/include/access/xlog.h b/src/include/access/xlog.h
index 77ac4e785f..1ec448c5d5 100644
--- a/src/include/access/xlog.h
+++ b/src/include/access/xlog.h
@@ -329,6 +329,7 @@ extern void InitXLOGAccess(void);
extern void CreateCheckPoint(int flags);
extern bool CreateRestartPoint(int flags);
extern WALAvailability GetWALAvailability(XLogRecPtr targetLSN);
+extern uint64 DistanceToWALHorizon(XLogRecPtr targetLSN, XLogRecPtr currptr);
extern XLogRecPtr CalculateMaxmumSafeLSN(void);
extern void XLogPutNextOid(Oid nextOid);
extern XLogRecPtr XLogRestorePoint(const char *rpName);
diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat
index 61f2c2f5b4..199fd994bd 100644
--- a/src/include/catalog/pg_proc.dat
+++ b/src/include/catalog/pg_proc.dat
@@ -10063,9 +10063,9 @@
proname => 'pg_get_replication_slots', prorows => '10', proisstrict => 'f',
proretset => 't', provolatile => 's', prorettype => 'record',
proargtypes => '',
- proallargtypes => '{name,name,text,oid,bool,bool,int4,xid,xid,pg_lsn,pg_lsn,text,pg_lsn}',
+ proallargtypes => '{name,name,text,oid,bool,bool,int4,xid,xid,pg_lsn,pg_lsn,text,int8}',
proargmodes => '{o,o,o,o,o,o,o,o,o,o,o,o,o}',
- proargnames => '{slot_name,plugin,slot_type,datoid,temporary,active,active_pid,xmin,catalog_xmin,restart_lsn,confirmed_flush_lsn,wal_status,min_safe_lsn}',
+ proargnames => '{slot_name,plugin,slot_type,datoid,temporary,active,active_pid,xmin,catalog_xmin,restart_lsn,confirmed_flush_lsn,wal_status,distance}',
prosrc => 'pg_get_replication_slots' },
{ oid => '3786', descr => 'set up a logical replication slot',
proname => 'pg_create_logical_replication_slot', provolatile => 'v',
diff --git a/src/test/recovery/t/019_replslot_limit.pl b/src/test/recovery/t/019_replslot_limit.pl
index 7d22ae5720..1c76d2d9e9 100644
--- a/src/test/recovery/t/019_replslot_limit.pl
+++ b/src/test/recovery/t/019_replslot_limit.pl
@@ -28,7 +28,7 @@ $node_master->safe_psql('postgres',
# The slot state and remain should be null before the first connection
my $result = $node_master->safe_psql('postgres',
- "SELECT restart_lsn IS NULL, wal_status is NULL, min_safe_lsn is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
+ "SELECT restart_lsn IS NULL, wal_status is NULL, distance is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
);
is($result, "t|t|t", 'check the state of non-reserved slot is "unknown"');
@@ -52,9 +52,9 @@ $node_master->wait_for_catchup($node_standby, 'replay', $start_lsn);
# Stop standby
$node_standby->stop;
-# Preparation done, the slot is the state "normal" now
+# Preparation done, the slot is the state "reserved" now
$result = $node_master->safe_psql('postgres',
- "SELECT wal_status, min_safe_lsn is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
+ "SELECT wal_status, distance is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
);
is($result, "reserved|t", 'check the catching-up state');
@@ -64,7 +64,7 @@ $node_master->safe_psql('postgres', "CHECKPOINT;");
# The slot is always "safe" when fitting max_wal_size
$result = $node_master->safe_psql('postgres',
- "SELECT wal_status, min_safe_lsn is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
+ "SELECT wal_status, distance is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
);
is($result, "reserved|t",
'check that it is safe if WAL fits in max_wal_size');
@@ -74,7 +74,7 @@ $node_master->safe_psql('postgres', "CHECKPOINT;");
# The slot is always "safe" when max_slot_wal_keep_size is not set
$result = $node_master->safe_psql('postgres',
- "SELECT wal_status, min_safe_lsn is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
+ "SELECT wal_status, distance is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
);
is($result, "reserved|t", 'check that slot is working');
@@ -94,9 +94,7 @@ max_slot_wal_keep_size = ${max_slot_wal_keep_size_mb}MB
));
$node_master->reload;
-# The slot is in safe state. The distance from the min_safe_lsn should
-# be as almost (max_slot_wal_keep_size - 1) times large as the segment
-# size
+# The slot is in safe state.
$result = $node_master->safe_psql('postgres',
"SELECT wal_status FROM pg_replication_slots WHERE slot_name = 'rep1'");
@@ -110,7 +108,7 @@ $node_master->safe_psql('postgres', "CHECKPOINT;");
$result = $node_master->safe_psql('postgres',
"SELECT wal_status FROM pg_replication_slots WHERE slot_name = 'rep1'");
is($result, "reserved",
- 'check that min_safe_lsn gets close to the current LSN');
+ 'check that distance gets close to the current LSN');
# The standby can reconnect to master
$node_standby->start;
@@ -154,7 +152,7 @@ advance_wal($node_master, 1);
# Slot gets into 'unreserved' state
$result = $node_master->safe_psql('postgres',
- "SELECT wal_status, min_safe_lsn is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
+ "SELECT wal_status, distance is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
);
is($result, "unreserved|t",
'check that the slot state changes to "unreserved"');
@@ -186,7 +184,7 @@ ok( find_in_log(
# This slot should be broken
$result = $node_master->safe_psql('postgres',
- "SELECT slot_name, active, restart_lsn IS NULL, wal_status, min_safe_lsn FROM pg_replication_slots WHERE slot_name = 'rep1'"
+ "SELECT slot_name, active, restart_lsn IS NULL, wal_status, distance FROM pg_replication_slots WHERE slot_name = 'rep1'"
);
is($result, "rep1|f|t|lost|",
'check that the slot became inactive and the state "lost" persists');
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b813e32215..392eab12dd 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1464,8 +1464,8 @@ pg_replication_slots| SELECT l.slot_name,
l.restart_lsn,
l.confirmed_flush_lsn,
l.wal_status,
- l.min_safe_lsn
- FROM (pg_get_replication_slots() l(slot_name, plugin, slot_type, datoid, temporary, active, active_pid, xmin, catalog_xmin, restart_lsn, confirmed_flush_lsn, wal_status, min_safe_lsn)
+ l.distance
+ FROM (pg_get_replication_slots() l(slot_name, plugin, slot_type, datoid, temporary, active, active_pid, xmin, catalog_xmin, restart_lsn, confirmed_flush_lsn, wal_status, distance)
LEFT JOIN pg_database d ON ((l.datoid = d.oid)));
pg_roles| SELECT pg_authid.rolname,
pg_authid.rolsuper,
--
2.18.4
----Next_Part(Wed_Jul__1_10_32_59_2020_161)----
^ permalink raw reply [nested|flat] 32+ messages in thread
* [PATCH v1] Replace min_safe_lsn with "distance" in pg_replication_slots
@ 2020-06-30 12:09 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 32+ messages in thread
From: Kyotaro Horiguchi @ 2020-06-30 12:09 UTC (permalink / raw)
pg_replication_slot.min_safe_lsn, which shows the oldest LSN kept in
pg_wal, is doubtful in usability for monitoring. Change it to
distance, which shows how many bytes the server can advance before the
slot loses required segments.
---
src/backend/access/transam/xlog.c | 39 +++++++++++++++++++++++
src/backend/catalog/system_views.sql | 2 +-
src/backend/replication/slotfuncs.c | 19 +++++------
src/include/access/xlog.h | 1 +
src/include/catalog/pg_proc.dat | 4 +--
src/test/recovery/t/019_replslot_limit.pl | 20 ++++++------
src/test/regress/expected/rules.out | 4 +--
7 files changed, 62 insertions(+), 27 deletions(-)
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index fd93bcfaeb..1f27639912 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -9570,6 +9570,45 @@ GetWALAvailability(XLogRecPtr targetLSN)
return WALAVAIL_REMOVED;
}
+/*
+ * Calculate how many bytes we can advance from currptr until the targetLSN is
+ * removed.
+ *
+ * Returns 0 if the distance is invalid.
+ */
+uint64
+DistanceToWALHorizon(XLogRecPtr targetLSN, XLogRecPtr currptr)
+{
+ XLogSegNo targetSeg;
+ XLogSegNo keepSegs;
+ XLogSegNo failSeg;
+ XLogRecPtr horizon;
+
+ XLByteToSeg(targetLSN, targetSeg, wal_segment_size);
+ keepSegs = 0;
+
+ /* no limit if max_slot_wal_keep_size is invalid */
+ if (max_slot_wal_keep_size_mb < 0)
+ return 0;
+
+ /* How many segments slots can keep? */
+ keepSegs = ConvertToXSegs(max_slot_wal_keep_size_mb, wal_segment_size);
+
+ /* override by wal_keep_segments if needed */
+ if (wal_keep_segments > keepSegs)
+ keepSegs = wal_keep_segments;
+
+ /* calculate the LSN where targetLSN is lost when currpos reaches */
+ failSeg = targetSeg + keepSegs + 1;
+ XLogSegNoOffsetToRecPtr(failSeg, 0, wal_segment_size, horizon);
+
+ /* If currptr already beyond the horizon, return zero. */
+ if (currptr > horizon)
+ return 0;
+
+ /* return the distance from currptr to the horizon */
+ return horizon - currptr;
+}
/*
* Retreat *logSegNo to the last segment that we need to retain because of
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index 5314e9348f..b9847a9f92 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -879,7 +879,7 @@ CREATE VIEW pg_replication_slots AS
L.restart_lsn,
L.confirmed_flush_lsn,
L.wal_status,
- L.min_safe_lsn
+ L.distance
FROM pg_get_replication_slots() AS L
LEFT JOIN pg_database D ON (L.datoid = D.oid);
diff --git a/src/backend/replication/slotfuncs.c b/src/backend/replication/slotfuncs.c
index 88033a79b2..532b3c5826 100644
--- a/src/backend/replication/slotfuncs.c
+++ b/src/backend/replication/slotfuncs.c
@@ -242,6 +242,7 @@ pg_get_replication_slots(PG_FUNCTION_ARGS)
Tuplestorestate *tupstore;
MemoryContext per_query_ctx;
MemoryContext oldcontext;
+ XLogRecPtr currlsn;
int slotno;
/* check to see if caller supports us returning a tuplestore */
@@ -274,6 +275,8 @@ pg_get_replication_slots(PG_FUNCTION_ARGS)
MemoryContextSwitchTo(oldcontext);
+ currlsn = GetXLogWriteRecPtr();
+
LWLockAcquire(ReplicationSlotControlLock, LW_SHARED);
for (slotno = 0; slotno < max_replication_slots; slotno++)
{
@@ -282,7 +285,7 @@ pg_get_replication_slots(PG_FUNCTION_ARGS)
Datum values[PG_GET_REPLICATION_SLOTS_COLS];
bool nulls[PG_GET_REPLICATION_SLOTS_COLS];
WALAvailability walstate;
- XLogSegNo last_removed_seg;
+ uint32 distance;
int i;
if (!slot->in_use)
@@ -398,16 +401,10 @@ pg_get_replication_slots(PG_FUNCTION_ARGS)
break;
}
- if (max_slot_wal_keep_size_mb >= 0 &&
- (walstate == WALAVAIL_RESERVED || walstate == WALAVAIL_EXTENDED) &&
- ((last_removed_seg = XLogGetLastRemovedSegno()) != 0))
- {
- XLogRecPtr min_safe_lsn;
-
- XLogSegNoOffsetToRecPtr(last_removed_seg + 1, 0,
- wal_segment_size, min_safe_lsn);
- values[i++] = Int64GetDatum(min_safe_lsn);
- }
+ distance =
+ DistanceToWALHorizon(slot_contents.data.restart_lsn, currlsn);
+ if (distance > 0)
+ values[i++] = Int64GetDatum(distance);
else
nulls[i++] = true;
diff --git a/src/include/access/xlog.h b/src/include/access/xlog.h
index 77ac4e785f..1ec448c5d5 100644
--- a/src/include/access/xlog.h
+++ b/src/include/access/xlog.h
@@ -329,6 +329,7 @@ extern void InitXLOGAccess(void);
extern void CreateCheckPoint(int flags);
extern bool CreateRestartPoint(int flags);
extern WALAvailability GetWALAvailability(XLogRecPtr targetLSN);
+extern uint64 DistanceToWALHorizon(XLogRecPtr targetLSN, XLogRecPtr currptr);
extern XLogRecPtr CalculateMaxmumSafeLSN(void);
extern void XLogPutNextOid(Oid nextOid);
extern XLogRecPtr XLogRestorePoint(const char *rpName);
diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat
index 61f2c2f5b4..199fd994bd 100644
--- a/src/include/catalog/pg_proc.dat
+++ b/src/include/catalog/pg_proc.dat
@@ -10063,9 +10063,9 @@
proname => 'pg_get_replication_slots', prorows => '10', proisstrict => 'f',
proretset => 't', provolatile => 's', prorettype => 'record',
proargtypes => '',
- proallargtypes => '{name,name,text,oid,bool,bool,int4,xid,xid,pg_lsn,pg_lsn,text,pg_lsn}',
+ proallargtypes => '{name,name,text,oid,bool,bool,int4,xid,xid,pg_lsn,pg_lsn,text,int8}',
proargmodes => '{o,o,o,o,o,o,o,o,o,o,o,o,o}',
- proargnames => '{slot_name,plugin,slot_type,datoid,temporary,active,active_pid,xmin,catalog_xmin,restart_lsn,confirmed_flush_lsn,wal_status,min_safe_lsn}',
+ proargnames => '{slot_name,plugin,slot_type,datoid,temporary,active,active_pid,xmin,catalog_xmin,restart_lsn,confirmed_flush_lsn,wal_status,distance}',
prosrc => 'pg_get_replication_slots' },
{ oid => '3786', descr => 'set up a logical replication slot',
proname => 'pg_create_logical_replication_slot', provolatile => 'v',
diff --git a/src/test/recovery/t/019_replslot_limit.pl b/src/test/recovery/t/019_replslot_limit.pl
index 7d22ae5720..1c76d2d9e9 100644
--- a/src/test/recovery/t/019_replslot_limit.pl
+++ b/src/test/recovery/t/019_replslot_limit.pl
@@ -28,7 +28,7 @@ $node_master->safe_psql('postgres',
# The slot state and remain should be null before the first connection
my $result = $node_master->safe_psql('postgres',
- "SELECT restart_lsn IS NULL, wal_status is NULL, min_safe_lsn is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
+ "SELECT restart_lsn IS NULL, wal_status is NULL, distance is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
);
is($result, "t|t|t", 'check the state of non-reserved slot is "unknown"');
@@ -52,9 +52,9 @@ $node_master->wait_for_catchup($node_standby, 'replay', $start_lsn);
# Stop standby
$node_standby->stop;
-# Preparation done, the slot is the state "normal" now
+# Preparation done, the slot is the state "reserved" now
$result = $node_master->safe_psql('postgres',
- "SELECT wal_status, min_safe_lsn is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
+ "SELECT wal_status, distance is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
);
is($result, "reserved|t", 'check the catching-up state');
@@ -64,7 +64,7 @@ $node_master->safe_psql('postgres', "CHECKPOINT;");
# The slot is always "safe" when fitting max_wal_size
$result = $node_master->safe_psql('postgres',
- "SELECT wal_status, min_safe_lsn is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
+ "SELECT wal_status, distance is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
);
is($result, "reserved|t",
'check that it is safe if WAL fits in max_wal_size');
@@ -74,7 +74,7 @@ $node_master->safe_psql('postgres', "CHECKPOINT;");
# The slot is always "safe" when max_slot_wal_keep_size is not set
$result = $node_master->safe_psql('postgres',
- "SELECT wal_status, min_safe_lsn is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
+ "SELECT wal_status, distance is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
);
is($result, "reserved|t", 'check that slot is working');
@@ -94,9 +94,7 @@ max_slot_wal_keep_size = ${max_slot_wal_keep_size_mb}MB
));
$node_master->reload;
-# The slot is in safe state. The distance from the min_safe_lsn should
-# be as almost (max_slot_wal_keep_size - 1) times large as the segment
-# size
+# The slot is in safe state.
$result = $node_master->safe_psql('postgres',
"SELECT wal_status FROM pg_replication_slots WHERE slot_name = 'rep1'");
@@ -110,7 +108,7 @@ $node_master->safe_psql('postgres', "CHECKPOINT;");
$result = $node_master->safe_psql('postgres',
"SELECT wal_status FROM pg_replication_slots WHERE slot_name = 'rep1'");
is($result, "reserved",
- 'check that min_safe_lsn gets close to the current LSN');
+ 'check that distance gets close to the current LSN');
# The standby can reconnect to master
$node_standby->start;
@@ -154,7 +152,7 @@ advance_wal($node_master, 1);
# Slot gets into 'unreserved' state
$result = $node_master->safe_psql('postgres',
- "SELECT wal_status, min_safe_lsn is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
+ "SELECT wal_status, distance is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
);
is($result, "unreserved|t",
'check that the slot state changes to "unreserved"');
@@ -186,7 +184,7 @@ ok( find_in_log(
# This slot should be broken
$result = $node_master->safe_psql('postgres',
- "SELECT slot_name, active, restart_lsn IS NULL, wal_status, min_safe_lsn FROM pg_replication_slots WHERE slot_name = 'rep1'"
+ "SELECT slot_name, active, restart_lsn IS NULL, wal_status, distance FROM pg_replication_slots WHERE slot_name = 'rep1'"
);
is($result, "rep1|f|t|lost|",
'check that the slot became inactive and the state "lost" persists');
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b813e32215..392eab12dd 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1464,8 +1464,8 @@ pg_replication_slots| SELECT l.slot_name,
l.restart_lsn,
l.confirmed_flush_lsn,
l.wal_status,
- l.min_safe_lsn
- FROM (pg_get_replication_slots() l(slot_name, plugin, slot_type, datoid, temporary, active, active_pid, xmin, catalog_xmin, restart_lsn, confirmed_flush_lsn, wal_status, min_safe_lsn)
+ l.distance
+ FROM (pg_get_replication_slots() l(slot_name, plugin, slot_type, datoid, temporary, active, active_pid, xmin, catalog_xmin, restart_lsn, confirmed_flush_lsn, wal_status, distance)
LEFT JOIN pg_database d ON ((l.datoid = d.oid)));
pg_roles| SELECT pg_authid.rolname,
pg_authid.rolsuper,
--
2.18.4
----Next_Part(Wed_Jul__1_10_32_59_2020_161)----
^ permalink raw reply [nested|flat] 32+ messages in thread
* [PATCH v1] Replace min_safe_lsn with "distance" in pg_replication_slots
@ 2020-06-30 12:09 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 32+ messages in thread
From: Kyotaro Horiguchi @ 2020-06-30 12:09 UTC (permalink / raw)
pg_replication_slot.min_safe_lsn, which shows the oldest LSN kept in
pg_wal, is doubtful in usability for monitoring. Change it to
distance, which shows how many bytes the server can advance before the
slot loses required segments.
---
src/backend/access/transam/xlog.c | 39 +++++++++++++++++++++++
src/backend/catalog/system_views.sql | 2 +-
src/backend/replication/slotfuncs.c | 19 +++++------
src/include/access/xlog.h | 1 +
src/include/catalog/pg_proc.dat | 4 +--
src/test/recovery/t/019_replslot_limit.pl | 20 ++++++------
src/test/regress/expected/rules.out | 4 +--
7 files changed, 62 insertions(+), 27 deletions(-)
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index fd93bcfaeb..1f27639912 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -9570,6 +9570,45 @@ GetWALAvailability(XLogRecPtr targetLSN)
return WALAVAIL_REMOVED;
}
+/*
+ * Calculate how many bytes we can advance from currptr until the targetLSN is
+ * removed.
+ *
+ * Returns 0 if the distance is invalid.
+ */
+uint64
+DistanceToWALHorizon(XLogRecPtr targetLSN, XLogRecPtr currptr)
+{
+ XLogSegNo targetSeg;
+ XLogSegNo keepSegs;
+ XLogSegNo failSeg;
+ XLogRecPtr horizon;
+
+ XLByteToSeg(targetLSN, targetSeg, wal_segment_size);
+ keepSegs = 0;
+
+ /* no limit if max_slot_wal_keep_size is invalid */
+ if (max_slot_wal_keep_size_mb < 0)
+ return 0;
+
+ /* How many segments slots can keep? */
+ keepSegs = ConvertToXSegs(max_slot_wal_keep_size_mb, wal_segment_size);
+
+ /* override by wal_keep_segments if needed */
+ if (wal_keep_segments > keepSegs)
+ keepSegs = wal_keep_segments;
+
+ /* calculate the LSN where targetLSN is lost when currpos reaches */
+ failSeg = targetSeg + keepSegs + 1;
+ XLogSegNoOffsetToRecPtr(failSeg, 0, wal_segment_size, horizon);
+
+ /* If currptr already beyond the horizon, return zero. */
+ if (currptr > horizon)
+ return 0;
+
+ /* return the distance from currptr to the horizon */
+ return horizon - currptr;
+}
/*
* Retreat *logSegNo to the last segment that we need to retain because of
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index 5314e9348f..b9847a9f92 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -879,7 +879,7 @@ CREATE VIEW pg_replication_slots AS
L.restart_lsn,
L.confirmed_flush_lsn,
L.wal_status,
- L.min_safe_lsn
+ L.distance
FROM pg_get_replication_slots() AS L
LEFT JOIN pg_database D ON (L.datoid = D.oid);
diff --git a/src/backend/replication/slotfuncs.c b/src/backend/replication/slotfuncs.c
index 88033a79b2..532b3c5826 100644
--- a/src/backend/replication/slotfuncs.c
+++ b/src/backend/replication/slotfuncs.c
@@ -242,6 +242,7 @@ pg_get_replication_slots(PG_FUNCTION_ARGS)
Tuplestorestate *tupstore;
MemoryContext per_query_ctx;
MemoryContext oldcontext;
+ XLogRecPtr currlsn;
int slotno;
/* check to see if caller supports us returning a tuplestore */
@@ -274,6 +275,8 @@ pg_get_replication_slots(PG_FUNCTION_ARGS)
MemoryContextSwitchTo(oldcontext);
+ currlsn = GetXLogWriteRecPtr();
+
LWLockAcquire(ReplicationSlotControlLock, LW_SHARED);
for (slotno = 0; slotno < max_replication_slots; slotno++)
{
@@ -282,7 +285,7 @@ pg_get_replication_slots(PG_FUNCTION_ARGS)
Datum values[PG_GET_REPLICATION_SLOTS_COLS];
bool nulls[PG_GET_REPLICATION_SLOTS_COLS];
WALAvailability walstate;
- XLogSegNo last_removed_seg;
+ uint32 distance;
int i;
if (!slot->in_use)
@@ -398,16 +401,10 @@ pg_get_replication_slots(PG_FUNCTION_ARGS)
break;
}
- if (max_slot_wal_keep_size_mb >= 0 &&
- (walstate == WALAVAIL_RESERVED || walstate == WALAVAIL_EXTENDED) &&
- ((last_removed_seg = XLogGetLastRemovedSegno()) != 0))
- {
- XLogRecPtr min_safe_lsn;
-
- XLogSegNoOffsetToRecPtr(last_removed_seg + 1, 0,
- wal_segment_size, min_safe_lsn);
- values[i++] = Int64GetDatum(min_safe_lsn);
- }
+ distance =
+ DistanceToWALHorizon(slot_contents.data.restart_lsn, currlsn);
+ if (distance > 0)
+ values[i++] = Int64GetDatum(distance);
else
nulls[i++] = true;
diff --git a/src/include/access/xlog.h b/src/include/access/xlog.h
index 77ac4e785f..1ec448c5d5 100644
--- a/src/include/access/xlog.h
+++ b/src/include/access/xlog.h
@@ -329,6 +329,7 @@ extern void InitXLOGAccess(void);
extern void CreateCheckPoint(int flags);
extern bool CreateRestartPoint(int flags);
extern WALAvailability GetWALAvailability(XLogRecPtr targetLSN);
+extern uint64 DistanceToWALHorizon(XLogRecPtr targetLSN, XLogRecPtr currptr);
extern XLogRecPtr CalculateMaxmumSafeLSN(void);
extern void XLogPutNextOid(Oid nextOid);
extern XLogRecPtr XLogRestorePoint(const char *rpName);
diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat
index 61f2c2f5b4..199fd994bd 100644
--- a/src/include/catalog/pg_proc.dat
+++ b/src/include/catalog/pg_proc.dat
@@ -10063,9 +10063,9 @@
proname => 'pg_get_replication_slots', prorows => '10', proisstrict => 'f',
proretset => 't', provolatile => 's', prorettype => 'record',
proargtypes => '',
- proallargtypes => '{name,name,text,oid,bool,bool,int4,xid,xid,pg_lsn,pg_lsn,text,pg_lsn}',
+ proallargtypes => '{name,name,text,oid,bool,bool,int4,xid,xid,pg_lsn,pg_lsn,text,int8}',
proargmodes => '{o,o,o,o,o,o,o,o,o,o,o,o,o}',
- proargnames => '{slot_name,plugin,slot_type,datoid,temporary,active,active_pid,xmin,catalog_xmin,restart_lsn,confirmed_flush_lsn,wal_status,min_safe_lsn}',
+ proargnames => '{slot_name,plugin,slot_type,datoid,temporary,active,active_pid,xmin,catalog_xmin,restart_lsn,confirmed_flush_lsn,wal_status,distance}',
prosrc => 'pg_get_replication_slots' },
{ oid => '3786', descr => 'set up a logical replication slot',
proname => 'pg_create_logical_replication_slot', provolatile => 'v',
diff --git a/src/test/recovery/t/019_replslot_limit.pl b/src/test/recovery/t/019_replslot_limit.pl
index 7d22ae5720..1c76d2d9e9 100644
--- a/src/test/recovery/t/019_replslot_limit.pl
+++ b/src/test/recovery/t/019_replslot_limit.pl
@@ -28,7 +28,7 @@ $node_master->safe_psql('postgres',
# The slot state and remain should be null before the first connection
my $result = $node_master->safe_psql('postgres',
- "SELECT restart_lsn IS NULL, wal_status is NULL, min_safe_lsn is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
+ "SELECT restart_lsn IS NULL, wal_status is NULL, distance is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
);
is($result, "t|t|t", 'check the state of non-reserved slot is "unknown"');
@@ -52,9 +52,9 @@ $node_master->wait_for_catchup($node_standby, 'replay', $start_lsn);
# Stop standby
$node_standby->stop;
-# Preparation done, the slot is the state "normal" now
+# Preparation done, the slot is the state "reserved" now
$result = $node_master->safe_psql('postgres',
- "SELECT wal_status, min_safe_lsn is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
+ "SELECT wal_status, distance is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
);
is($result, "reserved|t", 'check the catching-up state');
@@ -64,7 +64,7 @@ $node_master->safe_psql('postgres', "CHECKPOINT;");
# The slot is always "safe" when fitting max_wal_size
$result = $node_master->safe_psql('postgres',
- "SELECT wal_status, min_safe_lsn is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
+ "SELECT wal_status, distance is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
);
is($result, "reserved|t",
'check that it is safe if WAL fits in max_wal_size');
@@ -74,7 +74,7 @@ $node_master->safe_psql('postgres', "CHECKPOINT;");
# The slot is always "safe" when max_slot_wal_keep_size is not set
$result = $node_master->safe_psql('postgres',
- "SELECT wal_status, min_safe_lsn is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
+ "SELECT wal_status, distance is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
);
is($result, "reserved|t", 'check that slot is working');
@@ -94,9 +94,7 @@ max_slot_wal_keep_size = ${max_slot_wal_keep_size_mb}MB
));
$node_master->reload;
-# The slot is in safe state. The distance from the min_safe_lsn should
-# be as almost (max_slot_wal_keep_size - 1) times large as the segment
-# size
+# The slot is in safe state.
$result = $node_master->safe_psql('postgres',
"SELECT wal_status FROM pg_replication_slots WHERE slot_name = 'rep1'");
@@ -110,7 +108,7 @@ $node_master->safe_psql('postgres', "CHECKPOINT;");
$result = $node_master->safe_psql('postgres',
"SELECT wal_status FROM pg_replication_slots WHERE slot_name = 'rep1'");
is($result, "reserved",
- 'check that min_safe_lsn gets close to the current LSN');
+ 'check that distance gets close to the current LSN');
# The standby can reconnect to master
$node_standby->start;
@@ -154,7 +152,7 @@ advance_wal($node_master, 1);
# Slot gets into 'unreserved' state
$result = $node_master->safe_psql('postgres',
- "SELECT wal_status, min_safe_lsn is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
+ "SELECT wal_status, distance is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
);
is($result, "unreserved|t",
'check that the slot state changes to "unreserved"');
@@ -186,7 +184,7 @@ ok( find_in_log(
# This slot should be broken
$result = $node_master->safe_psql('postgres',
- "SELECT slot_name, active, restart_lsn IS NULL, wal_status, min_safe_lsn FROM pg_replication_slots WHERE slot_name = 'rep1'"
+ "SELECT slot_name, active, restart_lsn IS NULL, wal_status, distance FROM pg_replication_slots WHERE slot_name = 'rep1'"
);
is($result, "rep1|f|t|lost|",
'check that the slot became inactive and the state "lost" persists');
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b813e32215..392eab12dd 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1464,8 +1464,8 @@ pg_replication_slots| SELECT l.slot_name,
l.restart_lsn,
l.confirmed_flush_lsn,
l.wal_status,
- l.min_safe_lsn
- FROM (pg_get_replication_slots() l(slot_name, plugin, slot_type, datoid, temporary, active, active_pid, xmin, catalog_xmin, restart_lsn, confirmed_flush_lsn, wal_status, min_safe_lsn)
+ l.distance
+ FROM (pg_get_replication_slots() l(slot_name, plugin, slot_type, datoid, temporary, active, active_pid, xmin, catalog_xmin, restart_lsn, confirmed_flush_lsn, wal_status, distance)
LEFT JOIN pg_database d ON ((l.datoid = d.oid)));
pg_roles| SELECT pg_authid.rolname,
pg_authid.rolsuper,
--
2.18.4
----Next_Part(Wed_Jul__1_10_32_59_2020_161)----
^ permalink raw reply [nested|flat] 32+ messages in thread
* [PATCH v1] Replace min_safe_lsn with "distance" in pg_replication_slots
@ 2020-06-30 12:09 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 32+ messages in thread
From: Kyotaro Horiguchi @ 2020-06-30 12:09 UTC (permalink / raw)
pg_replication_slot.min_safe_lsn, which shows the oldest LSN kept in
pg_wal, is doubtful in usability for monitoring. Change it to
distance, which shows how many bytes the server can advance before the
slot loses required segments.
---
src/backend/access/transam/xlog.c | 39 +++++++++++++++++++++++
src/backend/catalog/system_views.sql | 2 +-
src/backend/replication/slotfuncs.c | 19 +++++------
src/include/access/xlog.h | 1 +
src/include/catalog/pg_proc.dat | 4 +--
src/test/recovery/t/019_replslot_limit.pl | 20 ++++++------
src/test/regress/expected/rules.out | 4 +--
7 files changed, 62 insertions(+), 27 deletions(-)
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index fd93bcfaeb..1f27639912 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -9570,6 +9570,45 @@ GetWALAvailability(XLogRecPtr targetLSN)
return WALAVAIL_REMOVED;
}
+/*
+ * Calculate how many bytes we can advance from currptr until the targetLSN is
+ * removed.
+ *
+ * Returns 0 if the distance is invalid.
+ */
+uint64
+DistanceToWALHorizon(XLogRecPtr targetLSN, XLogRecPtr currptr)
+{
+ XLogSegNo targetSeg;
+ XLogSegNo keepSegs;
+ XLogSegNo failSeg;
+ XLogRecPtr horizon;
+
+ XLByteToSeg(targetLSN, targetSeg, wal_segment_size);
+ keepSegs = 0;
+
+ /* no limit if max_slot_wal_keep_size is invalid */
+ if (max_slot_wal_keep_size_mb < 0)
+ return 0;
+
+ /* How many segments slots can keep? */
+ keepSegs = ConvertToXSegs(max_slot_wal_keep_size_mb, wal_segment_size);
+
+ /* override by wal_keep_segments if needed */
+ if (wal_keep_segments > keepSegs)
+ keepSegs = wal_keep_segments;
+
+ /* calculate the LSN where targetLSN is lost when currpos reaches */
+ failSeg = targetSeg + keepSegs + 1;
+ XLogSegNoOffsetToRecPtr(failSeg, 0, wal_segment_size, horizon);
+
+ /* If currptr already beyond the horizon, return zero. */
+ if (currptr > horizon)
+ return 0;
+
+ /* return the distance from currptr to the horizon */
+ return horizon - currptr;
+}
/*
* Retreat *logSegNo to the last segment that we need to retain because of
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index 5314e9348f..b9847a9f92 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -879,7 +879,7 @@ CREATE VIEW pg_replication_slots AS
L.restart_lsn,
L.confirmed_flush_lsn,
L.wal_status,
- L.min_safe_lsn
+ L.distance
FROM pg_get_replication_slots() AS L
LEFT JOIN pg_database D ON (L.datoid = D.oid);
diff --git a/src/backend/replication/slotfuncs.c b/src/backend/replication/slotfuncs.c
index 88033a79b2..532b3c5826 100644
--- a/src/backend/replication/slotfuncs.c
+++ b/src/backend/replication/slotfuncs.c
@@ -242,6 +242,7 @@ pg_get_replication_slots(PG_FUNCTION_ARGS)
Tuplestorestate *tupstore;
MemoryContext per_query_ctx;
MemoryContext oldcontext;
+ XLogRecPtr currlsn;
int slotno;
/* check to see if caller supports us returning a tuplestore */
@@ -274,6 +275,8 @@ pg_get_replication_slots(PG_FUNCTION_ARGS)
MemoryContextSwitchTo(oldcontext);
+ currlsn = GetXLogWriteRecPtr();
+
LWLockAcquire(ReplicationSlotControlLock, LW_SHARED);
for (slotno = 0; slotno < max_replication_slots; slotno++)
{
@@ -282,7 +285,7 @@ pg_get_replication_slots(PG_FUNCTION_ARGS)
Datum values[PG_GET_REPLICATION_SLOTS_COLS];
bool nulls[PG_GET_REPLICATION_SLOTS_COLS];
WALAvailability walstate;
- XLogSegNo last_removed_seg;
+ uint32 distance;
int i;
if (!slot->in_use)
@@ -398,16 +401,10 @@ pg_get_replication_slots(PG_FUNCTION_ARGS)
break;
}
- if (max_slot_wal_keep_size_mb >= 0 &&
- (walstate == WALAVAIL_RESERVED || walstate == WALAVAIL_EXTENDED) &&
- ((last_removed_seg = XLogGetLastRemovedSegno()) != 0))
- {
- XLogRecPtr min_safe_lsn;
-
- XLogSegNoOffsetToRecPtr(last_removed_seg + 1, 0,
- wal_segment_size, min_safe_lsn);
- values[i++] = Int64GetDatum(min_safe_lsn);
- }
+ distance =
+ DistanceToWALHorizon(slot_contents.data.restart_lsn, currlsn);
+ if (distance > 0)
+ values[i++] = Int64GetDatum(distance);
else
nulls[i++] = true;
diff --git a/src/include/access/xlog.h b/src/include/access/xlog.h
index 77ac4e785f..1ec448c5d5 100644
--- a/src/include/access/xlog.h
+++ b/src/include/access/xlog.h
@@ -329,6 +329,7 @@ extern void InitXLOGAccess(void);
extern void CreateCheckPoint(int flags);
extern bool CreateRestartPoint(int flags);
extern WALAvailability GetWALAvailability(XLogRecPtr targetLSN);
+extern uint64 DistanceToWALHorizon(XLogRecPtr targetLSN, XLogRecPtr currptr);
extern XLogRecPtr CalculateMaxmumSafeLSN(void);
extern void XLogPutNextOid(Oid nextOid);
extern XLogRecPtr XLogRestorePoint(const char *rpName);
diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat
index 61f2c2f5b4..199fd994bd 100644
--- a/src/include/catalog/pg_proc.dat
+++ b/src/include/catalog/pg_proc.dat
@@ -10063,9 +10063,9 @@
proname => 'pg_get_replication_slots', prorows => '10', proisstrict => 'f',
proretset => 't', provolatile => 's', prorettype => 'record',
proargtypes => '',
- proallargtypes => '{name,name,text,oid,bool,bool,int4,xid,xid,pg_lsn,pg_lsn,text,pg_lsn}',
+ proallargtypes => '{name,name,text,oid,bool,bool,int4,xid,xid,pg_lsn,pg_lsn,text,int8}',
proargmodes => '{o,o,o,o,o,o,o,o,o,o,o,o,o}',
- proargnames => '{slot_name,plugin,slot_type,datoid,temporary,active,active_pid,xmin,catalog_xmin,restart_lsn,confirmed_flush_lsn,wal_status,min_safe_lsn}',
+ proargnames => '{slot_name,plugin,slot_type,datoid,temporary,active,active_pid,xmin,catalog_xmin,restart_lsn,confirmed_flush_lsn,wal_status,distance}',
prosrc => 'pg_get_replication_slots' },
{ oid => '3786', descr => 'set up a logical replication slot',
proname => 'pg_create_logical_replication_slot', provolatile => 'v',
diff --git a/src/test/recovery/t/019_replslot_limit.pl b/src/test/recovery/t/019_replslot_limit.pl
index 7d22ae5720..1c76d2d9e9 100644
--- a/src/test/recovery/t/019_replslot_limit.pl
+++ b/src/test/recovery/t/019_replslot_limit.pl
@@ -28,7 +28,7 @@ $node_master->safe_psql('postgres',
# The slot state and remain should be null before the first connection
my $result = $node_master->safe_psql('postgres',
- "SELECT restart_lsn IS NULL, wal_status is NULL, min_safe_lsn is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
+ "SELECT restart_lsn IS NULL, wal_status is NULL, distance is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
);
is($result, "t|t|t", 'check the state of non-reserved slot is "unknown"');
@@ -52,9 +52,9 @@ $node_master->wait_for_catchup($node_standby, 'replay', $start_lsn);
# Stop standby
$node_standby->stop;
-# Preparation done, the slot is the state "normal" now
+# Preparation done, the slot is the state "reserved" now
$result = $node_master->safe_psql('postgres',
- "SELECT wal_status, min_safe_lsn is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
+ "SELECT wal_status, distance is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
);
is($result, "reserved|t", 'check the catching-up state');
@@ -64,7 +64,7 @@ $node_master->safe_psql('postgres', "CHECKPOINT;");
# The slot is always "safe" when fitting max_wal_size
$result = $node_master->safe_psql('postgres',
- "SELECT wal_status, min_safe_lsn is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
+ "SELECT wal_status, distance is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
);
is($result, "reserved|t",
'check that it is safe if WAL fits in max_wal_size');
@@ -74,7 +74,7 @@ $node_master->safe_psql('postgres', "CHECKPOINT;");
# The slot is always "safe" when max_slot_wal_keep_size is not set
$result = $node_master->safe_psql('postgres',
- "SELECT wal_status, min_safe_lsn is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
+ "SELECT wal_status, distance is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
);
is($result, "reserved|t", 'check that slot is working');
@@ -94,9 +94,7 @@ max_slot_wal_keep_size = ${max_slot_wal_keep_size_mb}MB
));
$node_master->reload;
-# The slot is in safe state. The distance from the min_safe_lsn should
-# be as almost (max_slot_wal_keep_size - 1) times large as the segment
-# size
+# The slot is in safe state.
$result = $node_master->safe_psql('postgres',
"SELECT wal_status FROM pg_replication_slots WHERE slot_name = 'rep1'");
@@ -110,7 +108,7 @@ $node_master->safe_psql('postgres', "CHECKPOINT;");
$result = $node_master->safe_psql('postgres',
"SELECT wal_status FROM pg_replication_slots WHERE slot_name = 'rep1'");
is($result, "reserved",
- 'check that min_safe_lsn gets close to the current LSN');
+ 'check that distance gets close to the current LSN');
# The standby can reconnect to master
$node_standby->start;
@@ -154,7 +152,7 @@ advance_wal($node_master, 1);
# Slot gets into 'unreserved' state
$result = $node_master->safe_psql('postgres',
- "SELECT wal_status, min_safe_lsn is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
+ "SELECT wal_status, distance is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'"
);
is($result, "unreserved|t",
'check that the slot state changes to "unreserved"');
@@ -186,7 +184,7 @@ ok( find_in_log(
# This slot should be broken
$result = $node_master->safe_psql('postgres',
- "SELECT slot_name, active, restart_lsn IS NULL, wal_status, min_safe_lsn FROM pg_replication_slots WHERE slot_name = 'rep1'"
+ "SELECT slot_name, active, restart_lsn IS NULL, wal_status, distance FROM pg_replication_slots WHERE slot_name = 'rep1'"
);
is($result, "rep1|f|t|lost|",
'check that the slot became inactive and the state "lost" persists');
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b813e32215..392eab12dd 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1464,8 +1464,8 @@ pg_replication_slots| SELECT l.slot_name,
l.restart_lsn,
l.confirmed_flush_lsn,
l.wal_status,
- l.min_safe_lsn
- FROM (pg_get_replication_slots() l(slot_name, plugin, slot_type, datoid, temporary, active, active_pid, xmin, catalog_xmin, restart_lsn, confirmed_flush_lsn, wal_status, min_safe_lsn)
+ l.distance
+ FROM (pg_get_replication_slots() l(slot_name, plugin, slot_type, datoid, temporary, active, active_pid, xmin, catalog_xmin, restart_lsn, confirmed_flush_lsn, wal_status, distance)
LEFT JOIN pg_database d ON ((l.datoid = d.oid)));
pg_roles| SELECT pg_authid.rolname,
pg_authid.rolsuper,
--
2.18.4
----Next_Part(Wed_Jul__1_10_32_59_2020_161)----
^ permalink raw reply [nested|flat] 32+ messages in thread
* Restrict copying of invalidated replication slots
@ 2025-02-04 09:56 Shlok Kyal <[email protected]>
0 siblings, 2 replies; 32+ messages in thread
From: Shlok Kyal @ 2025-02-04 09:56 UTC (permalink / raw)
To: PostgreSQL Hackers <[email protected]>
Hi,
Currently, we can copy an invalidated slot using the function
'pg_copy_logical_replication_slot'. As per the suggestion in the
thread [1], we should prohibit copying of such slots.
I have created a patch to address the issue.
[1]: https://www.postgresql.org/message-id/CAA4eK1Kw%3DvZ2FZ4DdrmOhuxOAL%3D2abaBm8hu_PsVN2Hd6UFP-w%40mail...
Thanks and Regards,
Shlok Kyal
Attachments:
[application/octet-stream] v1-0001-Restrict-copying-of-invalidated-replication-slots.patch (2.8K, ../../CANhcyEU65aH0VYnLiu=OhNNxhnhNhwcXBeT-jvRe1OiJTo_Ayg@mail.gmail.com/2-v1-0001-Restrict-copying-of-invalidated-replication-slots.patch)
download | inline diff:
From 349df8eada0d3eae63b009e34691240aad8cf258 Mon Sep 17 00:00:00 2001
From: Shlok Kyal <[email protected]>
Date: Tue, 4 Feb 2025 14:58:32 +0530
Subject: [PATCH v1] Restrict copying of invalidated replication slots
Currently we can copy an invalidated slot using function
'pg_copy_logical_replication_slot'. With this patch we will throw an
error in such case.
---
src/backend/replication/slotfuncs.c | 8 ++++++++
src/test/recovery/t/035_standby_logical_decoding.pl | 8 ++++++++
2 files changed, 16 insertions(+)
diff --git a/src/backend/replication/slotfuncs.c b/src/backend/replication/slotfuncs.c
index 8be4b8c65b..f83a890c50 100644
--- a/src/backend/replication/slotfuncs.c
+++ b/src/backend/replication/slotfuncs.c
@@ -604,6 +604,7 @@ copy_replication_slot(FunctionCallInfo fcinfo, bool logical_slot)
ReplicationSlot second_slot_contents;
XLogRecPtr src_restart_lsn;
bool src_islogical;
+ bool src_isinvalidated;
bool temporary;
char *plugin;
Datum values[2];
@@ -661,6 +662,7 @@ copy_replication_slot(FunctionCallInfo fcinfo, bool logical_slot)
src_restart_lsn = first_slot_contents.data.restart_lsn;
temporary = (first_slot_contents.data.persistency == RS_TEMPORARY);
plugin = logical_slot ? NameStr(first_slot_contents.data.plugin) : NULL;
+ src_isinvalidated = (first_slot_contents.data.invalidated != RS_INVAL_NONE);
/* Check type of replication slot */
if (src_islogical != logical_slot)
@@ -678,6 +680,12 @@ copy_replication_slot(FunctionCallInfo fcinfo, bool logical_slot)
(errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
errmsg("cannot copy a replication slot that doesn't reserve WAL")));
+ /* We should not copy invalidated replication slots */
+ if (src_isinvalidated)
+ ereport(ERROR,
+ (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
+ errmsg("cannot copy an invalidated replication slot")));
+
/* Overwrite params from optional arguments */
if (PG_NARGS() >= 3)
temporary = PG_GETARG_BOOL(2);
diff --git a/src/test/recovery/t/035_standby_logical_decoding.pl b/src/test/recovery/t/035_standby_logical_decoding.pl
index 505e85d1eb..efebf72bd6 100644
--- a/src/test/recovery/t/035_standby_logical_decoding.pl
+++ b/src/test/recovery/t/035_standby_logical_decoding.pl
@@ -553,6 +553,14 @@ $handle =
check_pg_recvlogical_stderr($handle,
"can no longer access replication slot \"vacuum_full_activeslot\"");
+# Attempting to copy an invalidated slot
+($result, $stdout, $stderr) = $node_standby->psql(
+ 'postgres',
+ qq[select pg_copy_logical_replication_slot('vacuum_full_inactiveslot', 'vacuum_full_inactiveslot_copy');],
+ replication => 'database');
+ok($stderr =~ /ERROR: cannot copy an invalidated replication slot/,
+ "invalidated slot cannot be copied");
+
# Turn hot_standby_feedback back on
change_hot_standby_feedback_and_wait_for_xmins(1, 1);
--
2.34.1
^ permalink raw reply [nested|flat] 32+ messages in thread
* Re: Restrict copying of invalidated replication slots
@ 2025-02-13 10:23 vignesh C <[email protected]>
parent: Shlok Kyal <[email protected]>
1 sibling, 1 reply; 32+ messages in thread
From: vignesh C @ 2025-02-13 10:23 UTC (permalink / raw)
To: Shlok Kyal <[email protected]>; +Cc: PostgreSQL Hackers <[email protected]>
On Tue, 4 Feb 2025 at 15:27, Shlok Kyal <[email protected]> wrote:
>
> Hi,
>
> Currently, we can copy an invalidated slot using the function
> 'pg_copy_logical_replication_slot'. As per the suggestion in the
> thread [1], we should prohibit copying of such slots.
>
> I have created a patch to address the issue.
This patch does not fix all the copy_replication_slot scenarios
completely, there is a very corner concurrency case where an
invalidated slot still gets copied:
+ /* We should not copy invalidated replication slots */
+ if (src_isinvalidated)
+ ereport(ERROR,
+
(errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
+ errmsg("cannot copy an invalidated
replication slot")));
Consider the following scenario:
step 1) Set up streaming replication between the primary and standby nodes.
step 2) Create a logical replication slot (test1) on the standby node.
step 3) Have a breakpoint in InvalidatePossiblyObsoleteSlot if cause
is RS_INVAL_WAL_LEVEL, no need to hold other invalidation causes or
add a sleep in InvalidatePossiblyObsoleteSlot function like below:
if (cause == RS_INVAL_WAL_LEVEL)
{
while (bsleep)
sleep(1);
}
step 4) Reduce wal_level on the primary to replica and restart the primary node.
step 5) SELECT 'copy' FROM pg_copy_logical_replication_slot('test1',
'test2'); -- It will wait till the lock held by
InvalidatePossiblyObsoleteSlot is released while trying to create a
slot.
step 6) Increase wal_level back to logical on the primary node and
restart the primary.
step 7) Now allow the invalidation to happen (continue the breakpoint
held at step 3), the replication control lock will be released and the
invalidated slot will be copied
After this:
postgres=# SELECT 'copy' FROM
pg_copy_logical_replication_slot('test1', 'test2');
?column?
----------
copy
(1 row)
-- The invalidated slot (test1) is copied successfully:
postgres=# select * from pg_replication_slots ;
slot_name | plugin | slot_type | datoid | database | temporary
| active | active_pid | xmin | catalog_xmin | restart_lsn |
confirmed_flush_lsn | wal_status | safe_wal_size | two_phas
e | inactive_since | conflicting |
invalidation_reason | failover | synced
-----------+---------------+-----------+--------+----------+-----------+--------+------------+------+--------------+-------------+---------------------+------------+---------------+---------
--+----------------------------------+-------------+------------------------+----------+--------
test1 | test_decoding | logical | 5 | postgres | f
| f | | | 745 | 0/4029060 | 0/4029098
| lost | | f
| 2025-02-13 15:26:54.666725+05:30 | t |
wal_level_insufficient | f | f
test2 | test_decoding | logical | 5 | postgres | f
| f | | | 745 | 0/4029060 | 0/4029098
| reserved | | f
| 2025-02-13 15:30:30.477836+05:30 | f |
| f | f
(2 rows)
-- A subsequent attempt to decode changes from the invalidated slot
(test2) fails:
postgres=# SELECT data FROM pg_logical_slot_get_changes('test2', NULL, NULL);
WARNING: detected write past chunk end in TXN 0x5e77e6c6f300
ERROR: logical decoding on standby requires "wal_level" >= "logical"
on the primary
-- Alternatively, the following error may occur:
postgres=# SELECT data FROM pg_logical_slot_get_changes('test2', NULL, NULL);
WARNING: detected write past chunk end in TXN 0x582d1b2d6ef0
data
------------
BEGIN 744
COMMIT 744
(2 rows)
This is an edge case that can occur under specific conditions
involving replication slot invalidation when there is a huge lag
between primary and standby.
There might be a similar concurrency case for wal_removed too.
Regards,
Vignesh
^ permalink raw reply [nested|flat] 32+ messages in thread
* Re: Restrict copying of invalidated replication slots
@ 2025-02-17 05:07 Peter Smith <[email protected]>
parent: Shlok Kyal <[email protected]>
1 sibling, 1 reply; 32+ messages in thread
From: Peter Smith @ 2025-02-17 05:07 UTC (permalink / raw)
To: Shlok Kyal <[email protected]>; +Cc: PostgreSQL Hackers <[email protected]>
Hi. Some review comments for patch v1-0001.
======
1. DOCS?
Shouldn't the documentation [1] for pg_copy_logical_replication_slot()
and pg_copy_physical_replication_slot() be updated to mention this?
======
src/backend/replication/slotfuncs.c
2.
+ /* We should not copy invalidated replication slots */
+ if (src_isinvalidated)
+ ereport(ERROR,
+ (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
+ errmsg("cannot copy an invalidated replication slot")));
+
2a.
The "we should not" sounds more like a recommendation than an error.
Comment can just say the same as the as errmsg.
~
2b.
ereport does not need all these parentheses
~
2c.
I felt the errmsg should include the name of the slot.
~~~
2d.
AFAICT this code will emit the same error regardless of
logical/physical slot so maybe you need to modify following to cater
for both kinds of replication_slot:
- the commit message
- docs
- test code
======
src/test/recovery/t/035_standby_logical_decoding.pl
3.
+# Attempting to copy an invalidated slot
+($result, $stdout, $stderr) = $node_standby->psql(
/Attempting/Attempt/
======
[1] https://www.postgresql.org/docs/current/functions-admin.html
Kind Regards,
Peter Smith.
Fujitsu Australia
^ permalink raw reply [nested|flat] 32+ messages in thread
* Re: Restrict copying of invalidated replication slots
@ 2025-02-17 11:31 Shlok Kyal <[email protected]>
parent: vignesh C <[email protected]>
0 siblings, 1 reply; 32+ messages in thread
From: Shlok Kyal @ 2025-02-17 11:31 UTC (permalink / raw)
To: vignesh C <[email protected]>; +Cc: PostgreSQL Hackers <[email protected]>
On Thu, 13 Feb 2025 at 15:54, vignesh C <[email protected]> wrote:
>
> On Tue, 4 Feb 2025 at 15:27, Shlok Kyal <[email protected]> wrote:
> >
> > Hi,
> >
> > Currently, we can copy an invalidated slot using the function
> > 'pg_copy_logical_replication_slot'. As per the suggestion in the
> > thread [1], we should prohibit copying of such slots.
> >
> > I have created a patch to address the issue.
>
> This patch does not fix all the copy_replication_slot scenarios
> completely, there is a very corner concurrency case where an
> invalidated slot still gets copied:
> + /* We should not copy invalidated replication slots */
> + if (src_isinvalidated)
> + ereport(ERROR,
> +
> (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
> + errmsg("cannot copy an invalidated
> replication slot")));
>
> Consider the following scenario:
> step 1) Set up streaming replication between the primary and standby nodes.
> step 2) Create a logical replication slot (test1) on the standby node.
> step 3) Have a breakpoint in InvalidatePossiblyObsoleteSlot if cause
> is RS_INVAL_WAL_LEVEL, no need to hold other invalidation causes or
> add a sleep in InvalidatePossiblyObsoleteSlot function like below:
> if (cause == RS_INVAL_WAL_LEVEL)
> {
> while (bsleep)
> sleep(1);
> }
> step 4) Reduce wal_level on the primary to replica and restart the primary node.
> step 5) SELECT 'copy' FROM pg_copy_logical_replication_slot('test1',
> 'test2'); -- It will wait till the lock held by
> InvalidatePossiblyObsoleteSlot is released while trying to create a
> slot.
> step 6) Increase wal_level back to logical on the primary node and
> restart the primary.
> step 7) Now allow the invalidation to happen (continue the breakpoint
> held at step 3), the replication control lock will be released and the
> invalidated slot will be copied
>
> After this:
> postgres=# SELECT 'copy' FROM
> pg_copy_logical_replication_slot('test1', 'test2');
> ?column?
> ----------
> copy
> (1 row)
>
> -- The invalidated slot (test1) is copied successfully:
> postgres=# select * from pg_replication_slots ;
> slot_name | plugin | slot_type | datoid | database | temporary
> | active | active_pid | xmin | catalog_xmin | restart_lsn |
> confirmed_flush_lsn | wal_status | safe_wal_size | two_phas
> e | inactive_since | conflicting |
> invalidation_reason | failover | synced
> -----------+---------------+-----------+--------+----------+-----------+--------+------------+------+--------------+-------------+---------------------+------------+---------------+---------
> --+----------------------------------+-------------+------------------------+----------+--------
> test1 | test_decoding | logical | 5 | postgres | f
> | f | | | 745 | 0/4029060 | 0/4029098
> | lost | | f
> | 2025-02-13 15:26:54.666725+05:30 | t |
> wal_level_insufficient | f | f
> test2 | test_decoding | logical | 5 | postgres | f
> | f | | | 745 | 0/4029060 | 0/4029098
> | reserved | | f
> | 2025-02-13 15:30:30.477836+05:30 | f |
> | f | f
> (2 rows)
>
> -- A subsequent attempt to decode changes from the invalidated slot
> (test2) fails:
> postgres=# SELECT data FROM pg_logical_slot_get_changes('test2', NULL, NULL);
> WARNING: detected write past chunk end in TXN 0x5e77e6c6f300
> ERROR: logical decoding on standby requires "wal_level" >= "logical"
> on the primary
>
> -- Alternatively, the following error may occur:
> postgres=# SELECT data FROM pg_logical_slot_get_changes('test2', NULL, NULL);
> WARNING: detected write past chunk end in TXN 0x582d1b2d6ef0
> data
> ------------
> BEGIN 744
> COMMIT 744
> (2 rows)
>
> This is an edge case that can occur under specific conditions
> involving replication slot invalidation when there is a huge lag
> between primary and standby.
> There might be a similar concurrency case for wal_removed too.
>
Hi Vignesh,
Thanks for reviewing the patch.
I have tested the above scenario and was able to reproduce it. I have
fixed it in the v2 patch.
Currently we are taking a shared lock on ReplicationSlotControlLock.
This issue can be resolved if we take an exclusive lock instead.
Thoughts?
Thanks and Regards,
Shlok Kyal
Attachments:
[application/octet-stream] v2-0001-Restrict-copying-of-invalidated-replication-slots.patch (4.5K, ../../CANhcyEVJpb6+hnk4MPVU3hZBYL=DS4v-PYBZOUoiivrN8Vd_Bw@mail.gmail.com/2-v2-0001-Restrict-copying-of-invalidated-replication-slots.patch)
download | inline diff:
From bb105104f249804a274c9cb2ffa9eaa9a0c41074 Mon Sep 17 00:00:00 2001
From: Shlok Kyal <[email protected]>
Date: Tue, 4 Feb 2025 14:58:32 +0530
Subject: [PATCH v2] Restrict copying of invalidated replication slots
Currently we can copy an invalidated logical and physical replication slot
using functions 'pg_copy_logical_replication_slot' and
'pg_copy_physical_replication_slot' respectively.
With this patch we will throw an error in such cases.
---
doc/src/sgml/func.sgml | 4 +++-
src/backend/replication/slotfuncs.c | 11 ++++++++++-
src/test/recovery/t/035_standby_logical_decoding.pl | 9 +++++++++
3 files changed, 22 insertions(+), 2 deletions(-)
diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml
index 7efc81936a..c88885e419 100644
--- a/doc/src/sgml/func.sgml
+++ b/doc/src/sgml/func.sgml
@@ -29342,7 +29342,8 @@ postgres=# SELECT '0/0'::pg_lsn + pd.segment_number * ps.setting::int + :offset
The copied physical slot starts to reserve WAL from the same <acronym>LSN</acronym> as the
source slot.
<parameter>temporary</parameter> is optional. If <parameter>temporary</parameter>
- is omitted, the same value as the source slot is used.
+ is omitted, the same value as the source slot is used. Copy of an
+ invalidated physical replication slot in not allowed.
</para></entry>
</row>
@@ -29364,6 +29365,7 @@ postgres=# SELECT '0/0'::pg_lsn + pd.segment_number * ps.setting::int + :offset
from the same <acronym>LSN</acronym> as the source logical slot. Both
<parameter>temporary</parameter> and <parameter>plugin</parameter> are
optional; if they are omitted, the values of the source slot are used.
+ Copy of an invalidated logical replication slot in not allowed.
</para></entry>
</row>
diff --git a/src/backend/replication/slotfuncs.c b/src/backend/replication/slotfuncs.c
index 8be4b8c65b..8a6365c1f8 100644
--- a/src/backend/replication/slotfuncs.c
+++ b/src/backend/replication/slotfuncs.c
@@ -604,6 +604,7 @@ copy_replication_slot(FunctionCallInfo fcinfo, bool logical_slot)
ReplicationSlot second_slot_contents;
XLogRecPtr src_restart_lsn;
bool src_islogical;
+ bool src_isinvalidated;
bool temporary;
char *plugin;
Datum values[2];
@@ -622,7 +623,7 @@ copy_replication_slot(FunctionCallInfo fcinfo, bool logical_slot)
else
CheckSlotRequirements();
- LWLockAcquire(ReplicationSlotControlLock, LW_SHARED);
+ LWLockAcquire(ReplicationSlotControlLock, LW_EXCLUSIVE);
/*
* We need to prevent the source slot's reserved WAL from being removed,
@@ -661,6 +662,7 @@ copy_replication_slot(FunctionCallInfo fcinfo, bool logical_slot)
src_restart_lsn = first_slot_contents.data.restart_lsn;
temporary = (first_slot_contents.data.persistency == RS_TEMPORARY);
plugin = logical_slot ? NameStr(first_slot_contents.data.plugin) : NULL;
+ src_isinvalidated = (first_slot_contents.data.invalidated != RS_INVAL_NONE);
/* Check type of replication slot */
if (src_islogical != logical_slot)
@@ -678,6 +680,13 @@ copy_replication_slot(FunctionCallInfo fcinfo, bool logical_slot)
(errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
errmsg("cannot copy a replication slot that doesn't reserve WAL")));
+ /* Cannot copy an invalidated replication slot */
+ if (src_isinvalidated)
+ ereport(ERROR,
+ errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
+ errmsg("cannot copy invalidated replication slot \"%s\"",
+ NameStr(*src_name)));
+
/* Overwrite params from optional arguments */
if (PG_NARGS() >= 3)
temporary = PG_GETARG_BOOL(2);
diff --git a/src/test/recovery/t/035_standby_logical_decoding.pl b/src/test/recovery/t/035_standby_logical_decoding.pl
index 505e85d1eb..23b235412a 100644
--- a/src/test/recovery/t/035_standby_logical_decoding.pl
+++ b/src/test/recovery/t/035_standby_logical_decoding.pl
@@ -553,6 +553,15 @@ $handle =
check_pg_recvlogical_stderr($handle,
"can no longer access replication slot \"vacuum_full_activeslot\"");
+# Attempt to copy an invalidated logical replication slot
+($result, $stdout, $stderr) = $node_standby->psql(
+ 'postgres',
+ qq[select pg_copy_logical_replication_slot('vacuum_full_inactiveslot', 'vacuum_full_inactiveslot_copy');],
+ replication => 'database');
+ok( $stderr =~
+ /ERROR: cannot copy invalidated replication slot "vacuum_full_inactiveslot"/,
+ "invalidated slot cannot be copied");
+
# Turn hot_standby_feedback back on
change_hot_standby_feedback_and_wait_for_xmins(1, 1);
--
2.34.1
^ permalink raw reply [nested|flat] 32+ messages in thread
* Re: Restrict copying of invalidated replication slots
@ 2025-02-17 11:34 Shlok Kyal <[email protected]>
parent: Peter Smith <[email protected]>
0 siblings, 0 replies; 32+ messages in thread
From: Shlok Kyal @ 2025-02-17 11:34 UTC (permalink / raw)
To: Peter Smith <[email protected]>; +Cc: PostgreSQL Hackers <[email protected]>
On Mon, 17 Feb 2025 at 10:37, Peter Smith <[email protected]> wrote:
>
> Hi. Some review comments for patch v1-0001.
>
> ======
> 1. DOCS?
>
> Shouldn't the documentation [1] for pg_copy_logical_replication_slot()
> and pg_copy_physical_replication_slot() be updated to mention this?
>
> ======
> src/backend/replication/slotfuncs.c
>
Updated the documentation
> 2.
> + /* We should not copy invalidated replication slots */
> + if (src_isinvalidated)
> + ereport(ERROR,
> + (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
> + errmsg("cannot copy an invalidated replication slot")));
> +
>
> 2a.
> The "we should not" sounds more like a recommendation than an error.
> Comment can just say the same as the as errmsg.
>
Fixed
> ~
>
> 2b.
> ereport does not need all these parentheses
>
Removed extra parentheses
> ~
>
> 2c.
> I felt the errmsg should include the name of the slot.
>
Added the slot name in error message
> ~~~
>
> 2d.
> AFAICT this code will emit the same error regardless of
> logical/physical slot so maybe you need to modify following to cater
> for both kinds of replication_slot:
> - the commit message
Fixed
> - docs
Fixed
> - test code
Currently a physical replication slot can only be invalidated for
"wal_removed". And logical replication slot can be invalidated for
"wal_removed", "rows_removed" , "wal_level_insufficient".
And for copying the slot invalidated for "wal_removed" throws error
"ERROR: cannot copy a replication slot that doesn't reserve WAL".
So, I have added test only for the case of logical replication slot.
>
> ======
> src/test/recovery/t/035_standby_logical_decoding.pl
>
> 3.
> +# Attempting to copy an invalidated slot
> +($result, $stdout, $stderr) = $node_standby->psql(
>
> /Attempting/Attempt/
>
Fixed
> ======
> [1] https://www.postgresql.org/docs/current/functions-admin.html
I have updated the changes in v2 patch [1].
[1]: https://www.postgresql.org/message-id/CANhcyEVJpb6%2Bhnk4MPVU3hZBYL%3DDS4v-PYBZOUoiivrN8Vd_Bw%40mail...
Thanks and Regards,
Shlok Kyal
^ permalink raw reply [nested|flat] 32+ messages in thread
* RE: Restrict copying of invalidated replication slots
@ 2025-02-18 09:55 Zhijie Hou (Fujitsu) <[email protected]>
parent: Shlok Kyal <[email protected]>
0 siblings, 1 reply; 32+ messages in thread
From: Zhijie Hou (Fujitsu) @ 2025-02-18 09:55 UTC (permalink / raw)
To: Shlok Kyal <[email protected]>; vignesh C <[email protected]>; +Cc: PostgreSQL Hackers <[email protected]>
On Monday, February 17, 2025 7:31 PM Shlok Kyal <[email protected]> wrote:
>
> On Thu, 13 Feb 2025 at 15:54, vignesh C <[email protected]> wrote:
> >
> > On Tue, 4 Feb 2025 at 15:27, Shlok Kyal <[email protected]> wrote:
> > >
> > > Hi,
> > >
> > > Currently, we can copy an invalidated slot using the function
> > > 'pg_copy_logical_replication_slot'. As per the suggestion in the
> > > thread [1], we should prohibit copying of such slots.
> > >
> > > I have created a patch to address the issue.
> >
> > This patch does not fix all the copy_replication_slot scenarios
> > completely, there is a very corner concurrency case where an
> > invalidated slot still gets copied:
> > + /* We should not copy invalidated replication slots */
> > + if (src_isinvalidated)
> > + ereport(ERROR,
> > +
> > (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
> > + errmsg("cannot copy an invalidated
> > replication slot")));
> >
> > Consider the following scenario:
> > step 1) Set up streaming replication between the primary and standby nodes.
> > step 2) Create a logical replication slot (test1) on the standby node.
> > step 3) Have a breakpoint in InvalidatePossiblyObsoleteSlot if cause
> > is RS_INVAL_WAL_LEVEL, no need to hold other invalidation causes or
> > add a sleep in InvalidatePossiblyObsoleteSlot function like below:
> > if (cause == RS_INVAL_WAL_LEVEL)
> > {
> > while (bsleep)
> > sleep(1);
> > }
> > step 4) Reduce wal_level on the primary to replica and restart the primary
> node.
> > step 5) SELECT 'copy' FROM pg_copy_logical_replication_slot('test1',
> > 'test2'); -- It will wait till the lock held by
> > InvalidatePossiblyObsoleteSlot is released while trying to create a
> > slot.
> > step 6) Increase wal_level back to logical on the primary node and
> > restart the primary.
> > step 7) Now allow the invalidation to happen (continue the breakpoint
> > held at step 3), the replication control lock will be released and the
> > invalidated slot will be copied
> >
> > After this:
> > postgres=# SELECT 'copy' FROM
> > pg_copy_logical_replication_slot('test1', 'test2'); ?column?
> > ----------
> > copy
> > (1 row)
> >
> > -- The invalidated slot (test1) is copied successfully:
> > postgres=# select * from pg_replication_slots ;
> > slot_name | plugin | slot_type | datoid | database | temporary
> > | active | active_pid | xmin | catalog_xmin | restart_lsn |
> > confirmed_flush_lsn | wal_status | safe_wal_size | two_phas
> > e | inactive_since | conflicting |
> > invalidation_reason | failover | synced
> >
> -----------+---------------+-----------+--------+----------+-----------+
> --------+------------+------+--------------+-------------+---------------
> ------+------------+---------------+---------
> >
> --+----------------------------------+-------------+----------------------
> --+----------+--------
> > test1 | test_decoding | logical | 5 | postgres | f
> > | f | | | 745 | 0/4029060 | 0/4029098
> > | lost | | f
> > | 2025-02-13 15:26:54.666725+05:30 | t |
> > wal_level_insufficient | f | f
> > test2 | test_decoding | logical | 5 | postgres | f
> > | f | | | 745 | 0/4029060 | 0/4029098
> > | reserved | | f
> > | 2025-02-13 15:30:30.477836+05:30 | f |
> > | f | f
> > (2 rows)
> >
> > -- A subsequent attempt to decode changes from the invalidated slot
> > (test2) fails:
> > postgres=# SELECT data FROM pg_logical_slot_get_changes('test2', NULL,
> > NULL);
> > WARNING: detected write past chunk end in TXN 0x5e77e6c6f300
> > ERROR: logical decoding on standby requires "wal_level" >= "logical"
> > on the primary
> >
> > -- Alternatively, the following error may occur:
> > postgres=# SELECT data FROM pg_logical_slot_get_changes('test2', NULL,
> > NULL);
> > WARNING: detected write past chunk end in TXN 0x582d1b2d6ef0
> > data
> > ------------
> > BEGIN 744
> > COMMIT 744
> > (2 rows)
> >
> > This is an edge case that can occur under specific conditions
> > involving replication slot invalidation when there is a huge lag
> > between primary and standby.
> > There might be a similar concurrency case for wal_removed too.
> >
>
> Hi Vignesh,
>
> Thanks for reviewing the patch.
Thanks for updating the patch. I have a question related to it.
>
> I have tested the above scenario and was able to reproduce it. I have fixed it in
> the v2 patch.
> Currently we are taking a shared lock on ReplicationSlotControlLock.
> This issue can be resolved if we take an exclusive lock instead.
> Thoughts?
It's not clear to me why increasing the lock level can solve it, could you
elaborate a bit more on this ?
Besides, do we need one more invalidated check in the following codes after
creating the slot ?
/*
* Check if the source slot still exists and is valid. We regard it as
* invalid if the type of replication slot or name has been changed,
* or the restart_lsn either is invalid or has gone backward. (The
...
Best Regards,
Hou zj
^ permalink raw reply [nested|flat] 32+ messages in thread
* Re: Restrict copying of invalidated replication slots
@ 2025-02-19 11:46 Shlok Kyal <[email protected]>
parent: Zhijie Hou (Fujitsu) <[email protected]>
0 siblings, 1 reply; 32+ messages in thread
From: Shlok Kyal @ 2025-02-19 11:46 UTC (permalink / raw)
To: Zhijie Hou (Fujitsu) <[email protected]>; +Cc: vignesh C <[email protected]>; PostgreSQL Hackers <[email protected]>
On Tue, 18 Feb 2025 at 15:26, Zhijie Hou (Fujitsu)
<[email protected]> wrote:
>
> On Monday, February 17, 2025 7:31 PM Shlok Kyal <[email protected]> wrote:
> >
> > On Thu, 13 Feb 2025 at 15:54, vignesh C <[email protected]> wrote:
> > >
> > > On Tue, 4 Feb 2025 at 15:27, Shlok Kyal <[email protected]> wrote:
> > > >
> > > > Hi,
> > > >
> > > > Currently, we can copy an invalidated slot using the function
> > > > 'pg_copy_logical_replication_slot'. As per the suggestion in the
> > > > thread [1], we should prohibit copying of such slots.
> > > >
> > > > I have created a patch to address the issue.
> > >
> > > This patch does not fix all the copy_replication_slot scenarios
> > > completely, there is a very corner concurrency case where an
> > > invalidated slot still gets copied:
> > > + /* We should not copy invalidated replication slots */
> > > + if (src_isinvalidated)
> > > + ereport(ERROR,
> > > +
> > > (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
> > > + errmsg("cannot copy an invalidated
> > > replication slot")));
> > >
> > > Consider the following scenario:
> > > step 1) Set up streaming replication between the primary and standby nodes.
> > > step 2) Create a logical replication slot (test1) on the standby node.
> > > step 3) Have a breakpoint in InvalidatePossiblyObsoleteSlot if cause
> > > is RS_INVAL_WAL_LEVEL, no need to hold other invalidation causes or
> > > add a sleep in InvalidatePossiblyObsoleteSlot function like below:
> > > if (cause == RS_INVAL_WAL_LEVEL)
> > > {
> > > while (bsleep)
> > > sleep(1);
> > > }
> > > step 4) Reduce wal_level on the primary to replica and restart the primary
> > node.
> > > step 5) SELECT 'copy' FROM pg_copy_logical_replication_slot('test1',
> > > 'test2'); -- It will wait till the lock held by
> > > InvalidatePossiblyObsoleteSlot is released while trying to create a
> > > slot.
> > > step 6) Increase wal_level back to logical on the primary node and
> > > restart the primary.
> > > step 7) Now allow the invalidation to happen (continue the breakpoint
> > > held at step 3), the replication control lock will be released and the
> > > invalidated slot will be copied
> > >
> > > After this:
> > > postgres=# SELECT 'copy' FROM
> > > pg_copy_logical_replication_slot('test1', 'test2'); ?column?
> > > ----------
> > > copy
> > > (1 row)
> > >
> > > -- The invalidated slot (test1) is copied successfully:
> > > postgres=# select * from pg_replication_slots ;
> > > slot_name | plugin | slot_type | datoid | database | temporary
> > > | active | active_pid | xmin | catalog_xmin | restart_lsn |
> > > confirmed_flush_lsn | wal_status | safe_wal_size | two_phas
> > > e | inactive_since | conflicting |
> > > invalidation_reason | failover | synced
> > >
> > -----------+---------------+-----------+--------+----------+-----------+
> > --------+------------+------+--------------+-------------+---------------
> > ------+------------+---------------+---------
> > >
> > --+----------------------------------+-------------+----------------------
> > --+----------+--------
> > > test1 | test_decoding | logical | 5 | postgres | f
> > > | f | | | 745 | 0/4029060 | 0/4029098
> > > | lost | | f
> > > | 2025-02-13 15:26:54.666725+05:30 | t |
> > > wal_level_insufficient | f | f
> > > test2 | test_decoding | logical | 5 | postgres | f
> > > | f | | | 745 | 0/4029060 | 0/4029098
> > > | reserved | | f
> > > | 2025-02-13 15:30:30.477836+05:30 | f |
> > > | f | f
> > > (2 rows)
> > >
> > > -- A subsequent attempt to decode changes from the invalidated slot
> > > (test2) fails:
> > > postgres=# SELECT data FROM pg_logical_slot_get_changes('test2', NULL,
> > > NULL);
> > > WARNING: detected write past chunk end in TXN 0x5e77e6c6f300
> > > ERROR: logical decoding on standby requires "wal_level" >= "logical"
> > > on the primary
> > >
> > > -- Alternatively, the following error may occur:
> > > postgres=# SELECT data FROM pg_logical_slot_get_changes('test2', NULL,
> > > NULL);
> > > WARNING: detected write past chunk end in TXN 0x582d1b2d6ef0
> > > data
> > > ------------
> > > BEGIN 744
> > > COMMIT 744
> > > (2 rows)
> > >
> > > This is an edge case that can occur under specific conditions
> > > involving replication slot invalidation when there is a huge lag
> > > between primary and standby.
> > > There might be a similar concurrency case for wal_removed too.
> > >
> >
> > Hi Vignesh,
> >
> > Thanks for reviewing the patch.
>
> Thanks for updating the patch. I have a question related to it.
>
> >
> > I have tested the above scenario and was able to reproduce it. I have fixed it in
> > the v2 patch.
> > Currently we are taking a shared lock on ReplicationSlotControlLock.
> > This issue can be resolved if we take an exclusive lock instead.
> > Thoughts?
>
> It's not clear to me why increasing the lock level can solve it, could you
> elaborate a bit more on this ?
>
In HEAD, InvalidateObsoleteReplicationSlots acquires a SHARED lock on
'ReplicationSlotControlLock'
Also in function 'copy_replication_slot' we take a SHARED lock on
'ReplicationSlotControlLock' during fetching of source slot.
So, for the case described by Vignesh in [1], first
InvalidateObsoleteReplicationSlot is called and we hold a SHARED lock
on 'ReplicationSlotControlLock'. We are now holding the function using
the sleep
if (cause == RS_INVAL_WAL_LEVEL)
{
while (bsleep)
sleep(1);
}
Now we create a copy of the slot since 'copy_replication_slot' takes
a SHARED lock on 'ReplicationSlotControlLock'. It will take the lock
and fetch the info of the source slot (the slot is not invalidated
till now). and the function 'copy_replication_slot' calls function
'create_logical_replication_slot' which takes a EXCLUSIVE lock on
ReplicationSlotControlLock and hence it will wait for function
InvalidateObsoleteReplicationSlot to release lock. Once the function
'InvalidateObsoleteReplicationSlot' releases the lock, the execution
of 'create_logical_replication_slot' continues and creates a copy of
the source slot.
Now with the patch, 'copy_replication_slot' will take an EXCLUSIVE
lock on 'ReplicationSlotControlLock'. to fetch the slot info. Hence,
it will wait for the 'InvalidateObsoleteReplicationSlot' to release
the lock and then fetch the source slot info and try to create the
copied slot (which will fail as source slot is invalidated before we
fetch its info)
> Besides, do we need one more invalidated check in the following codes after
> creating the slot ?
>
> /*
> * Check if the source slot still exists and is valid. We regard it as
> * invalid if the type of replication slot or name has been changed,
> * or the restart_lsn either is invalid or has gone backward. (The
> ...
>
This approach seems more feasible to me. It also resolves the issue
suggested by Vignesh in [1]. I have made changes for the same in v3
patch.
[1]: https://www.postgresql.org/message-id/CALDaNm2rrxO5mg6OKoScw84K5P1Tw_cbjniHm%2BGeyxme8Ei-nQ%40mail.g...
Thanks and Regards,
Shlok Kyal
Attachments:
[application/octet-stream] v3-0001-Restrict-copying-of-invalidated-replication-slots.patch (5.4K, ../../CANhcyEUAZe5piuYh3toK-ws-Y7rKCnamh36w2n1XzZQ0HJRU6A@mail.gmail.com/2-v3-0001-Restrict-copying-of-invalidated-replication-slots.patch)
download | inline diff:
From 629a0eab98e5f040f7b4b3f6d16a4b0ece8d0063 Mon Sep 17 00:00:00 2001
From: Shlok Kyal <[email protected]>
Date: Tue, 4 Feb 2025 14:58:32 +0530
Subject: [PATCH v3] Restrict copying of invalidated replication slots
Currently we can copy an invalidated logical and physical replication slot
using functions 'pg_copy_logical_replication_slot' and
'pg_copy_physical_replication_slot' respectively.
With this patch we will throw an error in such cases.
---
doc/src/sgml/func.sgml | 4 +-
src/backend/replication/slotfuncs.c | 37 +++++++++++++++++++
.../t/035_standby_logical_decoding.pl | 9 +++++
3 files changed, 49 insertions(+), 1 deletion(-)
diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml
index df32ee0bf5b..1bed0103723 100644
--- a/doc/src/sgml/func.sgml
+++ b/doc/src/sgml/func.sgml
@@ -29347,7 +29347,8 @@ postgres=# SELECT '0/0'::pg_lsn + pd.segment_number * ps.setting::int + :offset
The copied physical slot starts to reserve WAL from the same <acronym>LSN</acronym> as the
source slot.
<parameter>temporary</parameter> is optional. If <parameter>temporary</parameter>
- is omitted, the same value as the source slot is used.
+ is omitted, the same value as the source slot is used. Copy of an
+ invalidated physical replication slot in not allowed.
</para></entry>
</row>
@@ -29369,6 +29370,7 @@ postgres=# SELECT '0/0'::pg_lsn + pd.segment_number * ps.setting::int + :offset
from the same <acronym>LSN</acronym> as the source logical slot. Both
<parameter>temporary</parameter> and <parameter>plugin</parameter> are
optional; if they are omitted, the values of the source slot are used.
+ Copy of an invalidated logical replication slot in not allowed.
</para></entry>
</row>
diff --git a/src/backend/replication/slotfuncs.c b/src/backend/replication/slotfuncs.c
index f652ec8a73e..c845e16f80a 100644
--- a/src/backend/replication/slotfuncs.c
+++ b/src/backend/replication/slotfuncs.c
@@ -604,6 +604,7 @@ copy_replication_slot(FunctionCallInfo fcinfo, bool logical_slot)
ReplicationSlot second_slot_contents;
XLogRecPtr src_restart_lsn;
bool src_islogical;
+ bool src_isinvalidated;
bool temporary;
char *plugin;
Datum values[2];
@@ -661,6 +662,7 @@ copy_replication_slot(FunctionCallInfo fcinfo, bool logical_slot)
src_restart_lsn = first_slot_contents.data.restart_lsn;
temporary = (first_slot_contents.data.persistency == RS_TEMPORARY);
plugin = logical_slot ? NameStr(first_slot_contents.data.plugin) : NULL;
+ src_isinvalidated = (first_slot_contents.data.invalidated != RS_INVAL_NONE);
/* Check type of replication slot */
if (src_islogical != logical_slot)
@@ -678,6 +680,13 @@ copy_replication_slot(FunctionCallInfo fcinfo, bool logical_slot)
(errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
errmsg("cannot copy a replication slot that doesn't reserve WAL")));
+ /* Cannot copy an invalidated replication slot */
+ if (src_isinvalidated)
+ ereport(ERROR,
+ errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
+ errmsg("cannot copy invalidated replication slot \"%s\"",
+ NameStr(*src_name)));
+
/* Overwrite params from optional arguments */
if (PG_NARGS() >= 3)
temporary = PG_GETARG_BOOL(2);
@@ -774,6 +783,34 @@ copy_replication_slot(FunctionCallInfo fcinfo, bool logical_slot)
NameStr(*src_name)),
errhint("Retry when the source replication slot's confirmed_flush_lsn is valid.")));
+ /* Check if source slot was invalidated while copying of slot */
+ LWLockAcquire(ReplicationSlotControlLock, LW_SHARED);
+
+ for (int i = 0; i < max_replication_slots; i++)
+ {
+ ReplicationSlot *s = &ReplicationSlotCtl->replication_slots[i];
+
+ if (s->in_use && strcmp(NameStr(s->data.name), NameStr(*src_name)) == 0)
+ {
+ /* Copy the slot contents while holding spinlock */
+ SpinLockAcquire(&s->mutex);
+ first_slot_contents = *s;
+ SpinLockRelease(&s->mutex);
+ src = s;
+ break;
+ }
+ }
+
+ LWLockRelease(ReplicationSlotControlLock);
+
+ src_isinvalidated = (first_slot_contents.data.invalidated != RS_INVAL_NONE);
+
+ if (src_isinvalidated)
+ ereport(ERROR,
+ (errmsg("could not copy replication slot \"%s\"",
+ NameStr(*src_name)),
+ errdetail("The source replication slot was invalidated during the copy operation.")));
+
/* Install copied values again */
SpinLockAcquire(&MyReplicationSlot->mutex);
MyReplicationSlot->effective_xmin = copy_effective_xmin;
diff --git a/src/test/recovery/t/035_standby_logical_decoding.pl b/src/test/recovery/t/035_standby_logical_decoding.pl
index 505e85d1eb6..23b235412ac 100644
--- a/src/test/recovery/t/035_standby_logical_decoding.pl
+++ b/src/test/recovery/t/035_standby_logical_decoding.pl
@@ -553,6 +553,15 @@ $handle =
check_pg_recvlogical_stderr($handle,
"can no longer access replication slot \"vacuum_full_activeslot\"");
+# Attempt to copy an invalidated logical replication slot
+($result, $stdout, $stderr) = $node_standby->psql(
+ 'postgres',
+ qq[select pg_copy_logical_replication_slot('vacuum_full_inactiveslot', 'vacuum_full_inactiveslot_copy');],
+ replication => 'database');
+ok( $stderr =~
+ /ERROR: cannot copy invalidated replication slot "vacuum_full_inactiveslot"/,
+ "invalidated slot cannot be copied");
+
# Turn hot_standby_feedback back on
change_hot_standby_feedback_and_wait_for_xmins(1, 1);
--
2.34.1
^ permalink raw reply [nested|flat] 32+ messages in thread
* Re: Restrict copying of invalidated replication slots
@ 2025-02-20 19:43 Masahiko Sawada <[email protected]>
parent: Shlok Kyal <[email protected]>
0 siblings, 1 reply; 32+ messages in thread
From: Masahiko Sawada @ 2025-02-20 19:43 UTC (permalink / raw)
To: Shlok Kyal <[email protected]>; +Cc: Zhijie Hou (Fujitsu) <[email protected]>; vignesh C <[email protected]>; PostgreSQL Hackers <[email protected]>
On Wed, Feb 19, 2025 at 3:46 AM Shlok Kyal <[email protected]> wrote:
>
> On Tue, 18 Feb 2025 at 15:26, Zhijie Hou (Fujitsu)
> <[email protected]> wrote:
> >
> > On Monday, February 17, 2025 7:31 PM Shlok Kyal <[email protected]> wrote:
> > >
> > > On Thu, 13 Feb 2025 at 15:54, vignesh C <[email protected]> wrote:
> > > >
> > > > On Tue, 4 Feb 2025 at 15:27, Shlok Kyal <[email protected]> wrote:
> > > > >
> > > > > Hi,
> > > > >
> > > > > Currently, we can copy an invalidated slot using the function
> > > > > 'pg_copy_logical_replication_slot'. As per the suggestion in the
> > > > > thread [1], we should prohibit copying of such slots.
> > > > >
> > > > > I have created a patch to address the issue.
> > > >
> > > > This patch does not fix all the copy_replication_slot scenarios
> > > > completely, there is a very corner concurrency case where an
> > > > invalidated slot still gets copied:
> > > > + /* We should not copy invalidated replication slots */
> > > > + if (src_isinvalidated)
> > > > + ereport(ERROR,
> > > > +
> > > > (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
> > > > + errmsg("cannot copy an invalidated
> > > > replication slot")));
> > > >
> > > > Consider the following scenario:
> > > > step 1) Set up streaming replication between the primary and standby nodes.
> > > > step 2) Create a logical replication slot (test1) on the standby node.
> > > > step 3) Have a breakpoint in InvalidatePossiblyObsoleteSlot if cause
> > > > is RS_INVAL_WAL_LEVEL, no need to hold other invalidation causes or
> > > > add a sleep in InvalidatePossiblyObsoleteSlot function like below:
> > > > if (cause == RS_INVAL_WAL_LEVEL)
> > > > {
> > > > while (bsleep)
> > > > sleep(1);
> > > > }
> > > > step 4) Reduce wal_level on the primary to replica and restart the primary
> > > node.
> > > > step 5) SELECT 'copy' FROM pg_copy_logical_replication_slot('test1',
> > > > 'test2'); -- It will wait till the lock held by
> > > > InvalidatePossiblyObsoleteSlot is released while trying to create a
> > > > slot.
> > > > step 6) Increase wal_level back to logical on the primary node and
> > > > restart the primary.
> > > > step 7) Now allow the invalidation to happen (continue the breakpoint
> > > > held at step 3), the replication control lock will be released and the
> > > > invalidated slot will be copied
> > > >
> > > > After this:
> > > > postgres=# SELECT 'copy' FROM
> > > > pg_copy_logical_replication_slot('test1', 'test2'); ?column?
> > > > ----------
> > > > copy
> > > > (1 row)
> > > >
> > > > -- The invalidated slot (test1) is copied successfully:
> > > > postgres=# select * from pg_replication_slots ;
> > > > slot_name | plugin | slot_type | datoid | database | temporary
> > > > | active | active_pid | xmin | catalog_xmin | restart_lsn |
> > > > confirmed_flush_lsn | wal_status | safe_wal_size | two_phas
> > > > e | inactive_since | conflicting |
> > > > invalidation_reason | failover | synced
> > > >
> > > -----------+---------------+-----------+--------+----------+-----------+
> > > --------+------------+------+--------------+-------------+---------------
> > > ------+------------+---------------+---------
> > > >
> > > --+----------------------------------+-------------+----------------------
> > > --+----------+--------
> > > > test1 | test_decoding | logical | 5 | postgres | f
> > > > | f | | | 745 | 0/4029060 | 0/4029098
> > > > | lost | | f
> > > > | 2025-02-13 15:26:54.666725+05:30 | t |
> > > > wal_level_insufficient | f | f
> > > > test2 | test_decoding | logical | 5 | postgres | f
> > > > | f | | | 745 | 0/4029060 | 0/4029098
> > > > | reserved | | f
> > > > | 2025-02-13 15:30:30.477836+05:30 | f |
> > > > | f | f
> > > > (2 rows)
> > > >
> > > > -- A subsequent attempt to decode changes from the invalidated slot
> > > > (test2) fails:
> > > > postgres=# SELECT data FROM pg_logical_slot_get_changes('test2', NULL,
> > > > NULL);
> > > > WARNING: detected write past chunk end in TXN 0x5e77e6c6f300
> > > > ERROR: logical decoding on standby requires "wal_level" >= "logical"
> > > > on the primary
> > > >
> > > > -- Alternatively, the following error may occur:
> > > > postgres=# SELECT data FROM pg_logical_slot_get_changes('test2', NULL,
> > > > NULL);
> > > > WARNING: detected write past chunk end in TXN 0x582d1b2d6ef0
> > > > data
> > > > ------------
> > > > BEGIN 744
> > > > COMMIT 744
> > > > (2 rows)
> > > >
> > > > This is an edge case that can occur under specific conditions
> > > > involving replication slot invalidation when there is a huge lag
> > > > between primary and standby.
> > > > There might be a similar concurrency case for wal_removed too.
> > > >
> > >
> > > Hi Vignesh,
> > >
> > > Thanks for reviewing the patch.
> >
> > Thanks for updating the patch. I have a question related to it.
> >
> > >
> > > I have tested the above scenario and was able to reproduce it. I have fixed it in
> > > the v2 patch.
> > > Currently we are taking a shared lock on ReplicationSlotControlLock.
> > > This issue can be resolved if we take an exclusive lock instead.
> > > Thoughts?
> >
> > It's not clear to me why increasing the lock level can solve it, could you
> > elaborate a bit more on this ?
> >
> In HEAD, InvalidateObsoleteReplicationSlots acquires a SHARED lock on
> 'ReplicationSlotControlLock'
> Also in function 'copy_replication_slot' we take a SHARED lock on
> 'ReplicationSlotControlLock' during fetching of source slot.
>
> So, for the case described by Vignesh in [1], first
> InvalidateObsoleteReplicationSlot is called and we hold a SHARED lock
> on 'ReplicationSlotControlLock'. We are now holding the function using
> the sleep
> if (cause == RS_INVAL_WAL_LEVEL)
> {
> while (bsleep)
> sleep(1);
> }
>
> Now we create a copy of the slot since 'copy_replication_slot' takes
> a SHARED lock on 'ReplicationSlotControlLock'. It will take the lock
> and fetch the info of the source slot (the slot is not invalidated
> till now). and the function 'copy_replication_slot' calls function
> 'create_logical_replication_slot' which takes a EXCLUSIVE lock on
> ReplicationSlotControlLock and hence it will wait for function
> InvalidateObsoleteReplicationSlot to release lock. Once the function
> 'InvalidateObsoleteReplicationSlot' releases the lock, the execution
> of 'create_logical_replication_slot' continues and creates a copy of
> the source slot.
>
> Now with the patch, 'copy_replication_slot' will take an EXCLUSIVE
> lock on 'ReplicationSlotControlLock'. to fetch the slot info. Hence,
> it will wait for the 'InvalidateObsoleteReplicationSlot' to release
> the lock and then fetch the source slot info and try to create the
> copied slot (which will fail as source slot is invalidated before we
> fetch its info)
>
> > Besides, do we need one more invalidated check in the following codes after
> > creating the slot ?
> >
> > /*
> > * Check if the source slot still exists and is valid. We regard it as
> > * invalid if the type of replication slot or name has been changed,
> > * or the restart_lsn either is invalid or has gone backward. (The
> > ...
> >
>
> This approach seems more feasible to me. It also resolves the issue
> suggested by Vignesh in [1]. I have made changes for the same in v3
> patch.
>
I agree to check if the source slot got invalidated during the copy.
But why do we need to search the slot by the slot name again as
follows?
+ /* Check if source slot was invalidated while copying of slot */
+ LWLockAcquire(ReplicationSlotControlLock, LW_SHARED);
+
+ for (int i = 0; i < max_replication_slots; i++)
+ {
+ ReplicationSlot *s = &ReplicationSlotCtl->replication_slots[i];
+
+ if (s->in_use && strcmp(NameStr(s->data.name),
NameStr(*src_name)) == 0)
+ {
+ /* Copy the slot contents while holding spinlock */
+ SpinLockAcquire(&s->mutex);
+ first_slot_contents = *s;
+ SpinLockRelease(&s->mutex);
+ src = s;
+ break;
+ }
+ }
+
+ LWLockRelease(ReplicationSlotControlLock);
I think 'src' already points to the source slot.
Regards,
--
Masahiko Sawada
Amazon Web Services: https://aws.amazon.com
^ permalink raw reply [nested|flat] 32+ messages in thread
* Re: Restrict copying of invalidated replication slots
@ 2025-02-21 12:30 Shlok Kyal <[email protected]>
parent: Masahiko Sawada <[email protected]>
0 siblings, 2 replies; 32+ messages in thread
From: Shlok Kyal @ 2025-02-21 12:30 UTC (permalink / raw)
To: Masahiko Sawada <[email protected]>; +Cc: Zhijie Hou (Fujitsu) <[email protected]>; vignesh C <[email protected]>; PostgreSQL Hackers <[email protected]>
On Fri, 21 Feb 2025 at 01:14, Masahiko Sawada <[email protected]> wrote:
>
> On Wed, Feb 19, 2025 at 3:46 AM Shlok Kyal <[email protected]> wrote:
> >
> > On Tue, 18 Feb 2025 at 15:26, Zhijie Hou (Fujitsu)
> > <[email protected]> wrote:
> > >
> > > On Monday, February 17, 2025 7:31 PM Shlok Kyal <[email protected]> wrote:
> > > >
> > > > On Thu, 13 Feb 2025 at 15:54, vignesh C <[email protected]> wrote:
> > > > >
> > > > > On Tue, 4 Feb 2025 at 15:27, Shlok Kyal <[email protected]> wrote:
> > > > > >
> > > > > > Hi,
> > > > > >
> > > > > > Currently, we can copy an invalidated slot using the function
> > > > > > 'pg_copy_logical_replication_slot'. As per the suggestion in the
> > > > > > thread [1], we should prohibit copying of such slots.
> > > > > >
> > > > > > I have created a patch to address the issue.
> > > > >
> > > > > This patch does not fix all the copy_replication_slot scenarios
> > > > > completely, there is a very corner concurrency case where an
> > > > > invalidated slot still gets copied:
> > > > > + /* We should not copy invalidated replication slots */
> > > > > + if (src_isinvalidated)
> > > > > + ereport(ERROR,
> > > > > +
> > > > > (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
> > > > > + errmsg("cannot copy an invalidated
> > > > > replication slot")));
> > > > >
> > > > > Consider the following scenario:
> > > > > step 1) Set up streaming replication between the primary and standby nodes.
> > > > > step 2) Create a logical replication slot (test1) on the standby node.
> > > > > step 3) Have a breakpoint in InvalidatePossiblyObsoleteSlot if cause
> > > > > is RS_INVAL_WAL_LEVEL, no need to hold other invalidation causes or
> > > > > add a sleep in InvalidatePossiblyObsoleteSlot function like below:
> > > > > if (cause == RS_INVAL_WAL_LEVEL)
> > > > > {
> > > > > while (bsleep)
> > > > > sleep(1);
> > > > > }
> > > > > step 4) Reduce wal_level on the primary to replica and restart the primary
> > > > node.
> > > > > step 5) SELECT 'copy' FROM pg_copy_logical_replication_slot('test1',
> > > > > 'test2'); -- It will wait till the lock held by
> > > > > InvalidatePossiblyObsoleteSlot is released while trying to create a
> > > > > slot.
> > > > > step 6) Increase wal_level back to logical on the primary node and
> > > > > restart the primary.
> > > > > step 7) Now allow the invalidation to happen (continue the breakpoint
> > > > > held at step 3), the replication control lock will be released and the
> > > > > invalidated slot will be copied
> > > > >
> > > > > After this:
> > > > > postgres=# SELECT 'copy' FROM
> > > > > pg_copy_logical_replication_slot('test1', 'test2'); ?column?
> > > > > ----------
> > > > > copy
> > > > > (1 row)
> > > > >
> > > > > -- The invalidated slot (test1) is copied successfully:
> > > > > postgres=# select * from pg_replication_slots ;
> > > > > slot_name | plugin | slot_type | datoid | database | temporary
> > > > > | active | active_pid | xmin | catalog_xmin | restart_lsn |
> > > > > confirmed_flush_lsn | wal_status | safe_wal_size | two_phas
> > > > > e | inactive_since | conflicting |
> > > > > invalidation_reason | failover | synced
> > > > >
> > > > -----------+---------------+-----------+--------+----------+-----------+
> > > > --------+------------+------+--------------+-------------+---------------
> > > > ------+------------+---------------+---------
> > > > >
> > > > --+----------------------------------+-------------+----------------------
> > > > --+----------+--------
> > > > > test1 | test_decoding | logical | 5 | postgres | f
> > > > > | f | | | 745 | 0/4029060 | 0/4029098
> > > > > | lost | | f
> > > > > | 2025-02-13 15:26:54.666725+05:30 | t |
> > > > > wal_level_insufficient | f | f
> > > > > test2 | test_decoding | logical | 5 | postgres | f
> > > > > | f | | | 745 | 0/4029060 | 0/4029098
> > > > > | reserved | | f
> > > > > | 2025-02-13 15:30:30.477836+05:30 | f |
> > > > > | f | f
> > > > > (2 rows)
> > > > >
> > > > > -- A subsequent attempt to decode changes from the invalidated slot
> > > > > (test2) fails:
> > > > > postgres=# SELECT data FROM pg_logical_slot_get_changes('test2', NULL,
> > > > > NULL);
> > > > > WARNING: detected write past chunk end in TXN 0x5e77e6c6f300
> > > > > ERROR: logical decoding on standby requires "wal_level" >= "logical"
> > > > > on the primary
> > > > >
> > > > > -- Alternatively, the following error may occur:
> > > > > postgres=# SELECT data FROM pg_logical_slot_get_changes('test2', NULL,
> > > > > NULL);
> > > > > WARNING: detected write past chunk end in TXN 0x582d1b2d6ef0
> > > > > data
> > > > > ------------
> > > > > BEGIN 744
> > > > > COMMIT 744
> > > > > (2 rows)
> > > > >
> > > > > This is an edge case that can occur under specific conditions
> > > > > involving replication slot invalidation when there is a huge lag
> > > > > between primary and standby.
> > > > > There might be a similar concurrency case for wal_removed too.
> > > > >
> > > >
> > > > Hi Vignesh,
> > > >
> > > > Thanks for reviewing the patch.
> > >
> > > Thanks for updating the patch. I have a question related to it.
> > >
> > > >
> > > > I have tested the above scenario and was able to reproduce it. I have fixed it in
> > > > the v2 patch.
> > > > Currently we are taking a shared lock on ReplicationSlotControlLock.
> > > > This issue can be resolved if we take an exclusive lock instead.
> > > > Thoughts?
> > >
> > > It's not clear to me why increasing the lock level can solve it, could you
> > > elaborate a bit more on this ?
> > >
> > In HEAD, InvalidateObsoleteReplicationSlots acquires a SHARED lock on
> > 'ReplicationSlotControlLock'
> > Also in function 'copy_replication_slot' we take a SHARED lock on
> > 'ReplicationSlotControlLock' during fetching of source slot.
> >
> > So, for the case described by Vignesh in [1], first
> > InvalidateObsoleteReplicationSlot is called and we hold a SHARED lock
> > on 'ReplicationSlotControlLock'. We are now holding the function using
> > the sleep
> > if (cause == RS_INVAL_WAL_LEVEL)
> > {
> > while (bsleep)
> > sleep(1);
> > }
> >
> > Now we create a copy of the slot since 'copy_replication_slot' takes
> > a SHARED lock on 'ReplicationSlotControlLock'. It will take the lock
> > and fetch the info of the source slot (the slot is not invalidated
> > till now). and the function 'copy_replication_slot' calls function
> > 'create_logical_replication_slot' which takes a EXCLUSIVE lock on
> > ReplicationSlotControlLock and hence it will wait for function
> > InvalidateObsoleteReplicationSlot to release lock. Once the function
> > 'InvalidateObsoleteReplicationSlot' releases the lock, the execution
> > of 'create_logical_replication_slot' continues and creates a copy of
> > the source slot.
> >
> > Now with the patch, 'copy_replication_slot' will take an EXCLUSIVE
> > lock on 'ReplicationSlotControlLock'. to fetch the slot info. Hence,
> > it will wait for the 'InvalidateObsoleteReplicationSlot' to release
> > the lock and then fetch the source slot info and try to create the
> > copied slot (which will fail as source slot is invalidated before we
> > fetch its info)
> >
> > > Besides, do we need one more invalidated check in the following codes after
> > > creating the slot ?
> > >
> > > /*
> > > * Check if the source slot still exists and is valid. We regard it as
> > > * invalid if the type of replication slot or name has been changed,
> > > * or the restart_lsn either is invalid or has gone backward. (The
> > > ...
> > >
> >
> > This approach seems more feasible to me. It also resolves the issue
> > suggested by Vignesh in [1]. I have made changes for the same in v3
> > patch.
> >
>
> I agree to check if the source slot got invalidated during the copy.
> But why do we need to search the slot by the slot name again as
> follows?
>
> + /* Check if source slot was invalidated while copying of slot */
> + LWLockAcquire(ReplicationSlotControlLock, LW_SHARED);
> +
> + for (int i = 0; i < max_replication_slots; i++)
> + {
> + ReplicationSlot *s = &ReplicationSlotCtl->replication_slots[i];
> +
> + if (s->in_use && strcmp(NameStr(s->data.name),
> NameStr(*src_name)) == 0)
> + {
> + /* Copy the slot contents while holding spinlock */
> + SpinLockAcquire(&s->mutex);
> + first_slot_contents = *s;
> + SpinLockRelease(&s->mutex);
> + src = s;
> + break;
> + }
> + }
> +
> + LWLockRelease(ReplicationSlotControlLock);
>
> I think 'src' already points to the source slot.
>
Hi Sawada san,
Thanks for reviewing the patch.
I have used the 'src' instead of iterating again. I have attached the
updated v4 patch.
Thanks and Regards,
Shlok Kyal
Attachments:
[application/octet-stream] v4-0001-Restrict-copying-of-invalidated-replication-slots.patch (5.0K, ../../CANhcyEWJ=bfi5FSEVtvkTRMd5z7gVNG+nxEDPLengtNprCvfoQ@mail.gmail.com/2-v4-0001-Restrict-copying-of-invalidated-replication-slots.patch)
download | inline diff:
From a5e1d31ed2fdb30ac8c87c7dad9da935e9df07d6 Mon Sep 17 00:00:00 2001
From: Shlok Kyal <[email protected]>
Date: Tue, 4 Feb 2025 14:58:32 +0530
Subject: [PATCH v4] Restrict copying of invalidated replication slots
Currently we can copy an invalidated logical and physical replication slot
using functions 'pg_copy_logical_replication_slot' and
'pg_copy_physical_replication_slot' respectively.
With this patch we will throw an error in such cases.
---
doc/src/sgml/func.sgml | 4 +++-
src/backend/replication/slotfuncs.c | 22 +++++++++++++++++++
.../t/035_standby_logical_decoding.pl | 9 ++++++++
3 files changed, 34 insertions(+), 1 deletion(-)
diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml
index df32ee0bf5b..1bed0103723 100644
--- a/doc/src/sgml/func.sgml
+++ b/doc/src/sgml/func.sgml
@@ -29347,7 +29347,8 @@ postgres=# SELECT '0/0'::pg_lsn + pd.segment_number * ps.setting::int + :offset
The copied physical slot starts to reserve WAL from the same <acronym>LSN</acronym> as the
source slot.
<parameter>temporary</parameter> is optional. If <parameter>temporary</parameter>
- is omitted, the same value as the source slot is used.
+ is omitted, the same value as the source slot is used. Copy of an
+ invalidated physical replication slot in not allowed.
</para></entry>
</row>
@@ -29369,6 +29370,7 @@ postgres=# SELECT '0/0'::pg_lsn + pd.segment_number * ps.setting::int + :offset
from the same <acronym>LSN</acronym> as the source logical slot. Both
<parameter>temporary</parameter> and <parameter>plugin</parameter> are
optional; if they are omitted, the values of the source slot are used.
+ Copy of an invalidated logical replication slot in not allowed.
</para></entry>
</row>
diff --git a/src/backend/replication/slotfuncs.c b/src/backend/replication/slotfuncs.c
index f652ec8a73e..7669604cbbb 100644
--- a/src/backend/replication/slotfuncs.c
+++ b/src/backend/replication/slotfuncs.c
@@ -604,6 +604,7 @@ copy_replication_slot(FunctionCallInfo fcinfo, bool logical_slot)
ReplicationSlot second_slot_contents;
XLogRecPtr src_restart_lsn;
bool src_islogical;
+ bool src_isinvalidated;
bool temporary;
char *plugin;
Datum values[2];
@@ -661,6 +662,7 @@ copy_replication_slot(FunctionCallInfo fcinfo, bool logical_slot)
src_restart_lsn = first_slot_contents.data.restart_lsn;
temporary = (first_slot_contents.data.persistency == RS_TEMPORARY);
plugin = logical_slot ? NameStr(first_slot_contents.data.plugin) : NULL;
+ src_isinvalidated = (first_slot_contents.data.invalidated != RS_INVAL_NONE);
/* Check type of replication slot */
if (src_islogical != logical_slot)
@@ -678,6 +680,13 @@ copy_replication_slot(FunctionCallInfo fcinfo, bool logical_slot)
(errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
errmsg("cannot copy a replication slot that doesn't reserve WAL")));
+ /* Cannot copy an invalidated replication slot */
+ if (src_isinvalidated)
+ ereport(ERROR,
+ errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
+ errmsg("cannot copy invalidated replication slot \"%s\"",
+ NameStr(*src_name)));
+
/* Overwrite params from optional arguments */
if (PG_NARGS() >= 3)
temporary = PG_GETARG_BOOL(2);
@@ -774,6 +783,19 @@ copy_replication_slot(FunctionCallInfo fcinfo, bool logical_slot)
NameStr(*src_name)),
errhint("Retry when the source replication slot's confirmed_flush_lsn is valid.")));
+ /* Check if source slot was invalidated while copying of slot */
+ SpinLockAcquire(&src->mutex);
+ first_slot_contents = *src;
+ SpinLockRelease(&src->mutex);
+
+ src_isinvalidated = (first_slot_contents.data.invalidated != RS_INVAL_NONE);
+
+ if (src_isinvalidated)
+ ereport(ERROR,
+ (errmsg("could not copy replication slot \"%s\"",
+ NameStr(*src_name)),
+ errdetail("The source replication slot was invalidated during the copy operation.")));
+
/* Install copied values again */
SpinLockAcquire(&MyReplicationSlot->mutex);
MyReplicationSlot->effective_xmin = copy_effective_xmin;
diff --git a/src/test/recovery/t/035_standby_logical_decoding.pl b/src/test/recovery/t/035_standby_logical_decoding.pl
index 505e85d1eb6..23b235412ac 100644
--- a/src/test/recovery/t/035_standby_logical_decoding.pl
+++ b/src/test/recovery/t/035_standby_logical_decoding.pl
@@ -553,6 +553,15 @@ $handle =
check_pg_recvlogical_stderr($handle,
"can no longer access replication slot \"vacuum_full_activeslot\"");
+# Attempt to copy an invalidated logical replication slot
+($result, $stdout, $stderr) = $node_standby->psql(
+ 'postgres',
+ qq[select pg_copy_logical_replication_slot('vacuum_full_inactiveslot', 'vacuum_full_inactiveslot_copy');],
+ replication => 'database');
+ok( $stderr =~
+ /ERROR: cannot copy invalidated replication slot "vacuum_full_inactiveslot"/,
+ "invalidated slot cannot be copied");
+
# Turn hot_standby_feedback back on
change_hot_standby_feedback_and_wait_for_xmins(1, 1);
--
2.34.1
^ permalink raw reply [nested|flat] 32+ messages in thread
* Re: Restrict copying of invalidated replication slots
@ 2025-02-21 23:18 Masahiko Sawada <[email protected]>
parent: Shlok Kyal <[email protected]>
1 sibling, 0 replies; 32+ messages in thread
From: Masahiko Sawada @ 2025-02-21 23:18 UTC (permalink / raw)
To: Shlok Kyal <[email protected]>; +Cc: Zhijie Hou (Fujitsu) <[email protected]>; vignesh C <[email protected]>; PostgreSQL Hackers <[email protected]>
On Fri, Feb 21, 2025 at 4:30 AM Shlok Kyal <[email protected]> wrote:
>
> On Fri, 21 Feb 2025 at 01:14, Masahiko Sawada <[email protected]> wrote:
> >
> > On Wed, Feb 19, 2025 at 3:46 AM Shlok Kyal <[email protected]> wrote:
> > >
> > > On Tue, 18 Feb 2025 at 15:26, Zhijie Hou (Fujitsu)
> > > <[email protected]> wrote:
> > > >
> > > > On Monday, February 17, 2025 7:31 PM Shlok Kyal <[email protected]> wrote:
> > > > >
> > > > > On Thu, 13 Feb 2025 at 15:54, vignesh C <[email protected]> wrote:
> > > > > >
> > > > > > On Tue, 4 Feb 2025 at 15:27, Shlok Kyal <[email protected]> wrote:
> > > > > > >
> > > > > > > Hi,
> > > > > > >
> > > > > > > Currently, we can copy an invalidated slot using the function
> > > > > > > 'pg_copy_logical_replication_slot'. As per the suggestion in the
> > > > > > > thread [1], we should prohibit copying of such slots.
> > > > > > >
> > > > > > > I have created a patch to address the issue.
> > > > > >
> > > > > > This patch does not fix all the copy_replication_slot scenarios
> > > > > > completely, there is a very corner concurrency case where an
> > > > > > invalidated slot still gets copied:
> > > > > > + /* We should not copy invalidated replication slots */
> > > > > > + if (src_isinvalidated)
> > > > > > + ereport(ERROR,
> > > > > > +
> > > > > > (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
> > > > > > + errmsg("cannot copy an invalidated
> > > > > > replication slot")));
> > > > > >
> > > > > > Consider the following scenario:
> > > > > > step 1) Set up streaming replication between the primary and standby nodes.
> > > > > > step 2) Create a logical replication slot (test1) on the standby node.
> > > > > > step 3) Have a breakpoint in InvalidatePossiblyObsoleteSlot if cause
> > > > > > is RS_INVAL_WAL_LEVEL, no need to hold other invalidation causes or
> > > > > > add a sleep in InvalidatePossiblyObsoleteSlot function like below:
> > > > > > if (cause == RS_INVAL_WAL_LEVEL)
> > > > > > {
> > > > > > while (bsleep)
> > > > > > sleep(1);
> > > > > > }
> > > > > > step 4) Reduce wal_level on the primary to replica and restart the primary
> > > > > node.
> > > > > > step 5) SELECT 'copy' FROM pg_copy_logical_replication_slot('test1',
> > > > > > 'test2'); -- It will wait till the lock held by
> > > > > > InvalidatePossiblyObsoleteSlot is released while trying to create a
> > > > > > slot.
> > > > > > step 6) Increase wal_level back to logical on the primary node and
> > > > > > restart the primary.
> > > > > > step 7) Now allow the invalidation to happen (continue the breakpoint
> > > > > > held at step 3), the replication control lock will be released and the
> > > > > > invalidated slot will be copied
> > > > > >
> > > > > > After this:
> > > > > > postgres=# SELECT 'copy' FROM
> > > > > > pg_copy_logical_replication_slot('test1', 'test2'); ?column?
> > > > > > ----------
> > > > > > copy
> > > > > > (1 row)
> > > > > >
> > > > > > -- The invalidated slot (test1) is copied successfully:
> > > > > > postgres=# select * from pg_replication_slots ;
> > > > > > slot_name | plugin | slot_type | datoid | database | temporary
> > > > > > | active | active_pid | xmin | catalog_xmin | restart_lsn |
> > > > > > confirmed_flush_lsn | wal_status | safe_wal_size | two_phas
> > > > > > e | inactive_since | conflicting |
> > > > > > invalidation_reason | failover | synced
> > > > > >
> > > > > -----------+---------------+-----------+--------+----------+-----------+
> > > > > --------+------------+------+--------------+-------------+---------------
> > > > > ------+------------+---------------+---------
> > > > > >
> > > > > --+----------------------------------+-------------+----------------------
> > > > > --+----------+--------
> > > > > > test1 | test_decoding | logical | 5 | postgres | f
> > > > > > | f | | | 745 | 0/4029060 | 0/4029098
> > > > > > | lost | | f
> > > > > > | 2025-02-13 15:26:54.666725+05:30 | t |
> > > > > > wal_level_insufficient | f | f
> > > > > > test2 | test_decoding | logical | 5 | postgres | f
> > > > > > | f | | | 745 | 0/4029060 | 0/4029098
> > > > > > | reserved | | f
> > > > > > | 2025-02-13 15:30:30.477836+05:30 | f |
> > > > > > | f | f
> > > > > > (2 rows)
> > > > > >
> > > > > > -- A subsequent attempt to decode changes from the invalidated slot
> > > > > > (test2) fails:
> > > > > > postgres=# SELECT data FROM pg_logical_slot_get_changes('test2', NULL,
> > > > > > NULL);
> > > > > > WARNING: detected write past chunk end in TXN 0x5e77e6c6f300
> > > > > > ERROR: logical decoding on standby requires "wal_level" >= "logical"
> > > > > > on the primary
> > > > > >
> > > > > > -- Alternatively, the following error may occur:
> > > > > > postgres=# SELECT data FROM pg_logical_slot_get_changes('test2', NULL,
> > > > > > NULL);
> > > > > > WARNING: detected write past chunk end in TXN 0x582d1b2d6ef0
> > > > > > data
> > > > > > ------------
> > > > > > BEGIN 744
> > > > > > COMMIT 744
> > > > > > (2 rows)
> > > > > >
> > > > > > This is an edge case that can occur under specific conditions
> > > > > > involving replication slot invalidation when there is a huge lag
> > > > > > between primary and standby.
> > > > > > There might be a similar concurrency case for wal_removed too.
> > > > > >
> > > > >
> > > > > Hi Vignesh,
> > > > >
> > > > > Thanks for reviewing the patch.
> > > >
> > > > Thanks for updating the patch. I have a question related to it.
> > > >
> > > > >
> > > > > I have tested the above scenario and was able to reproduce it. I have fixed it in
> > > > > the v2 patch.
> > > > > Currently we are taking a shared lock on ReplicationSlotControlLock.
> > > > > This issue can be resolved if we take an exclusive lock instead.
> > > > > Thoughts?
> > > >
> > > > It's not clear to me why increasing the lock level can solve it, could you
> > > > elaborate a bit more on this ?
> > > >
> > > In HEAD, InvalidateObsoleteReplicationSlots acquires a SHARED lock on
> > > 'ReplicationSlotControlLock'
> > > Also in function 'copy_replication_slot' we take a SHARED lock on
> > > 'ReplicationSlotControlLock' during fetching of source slot.
> > >
> > > So, for the case described by Vignesh in [1], first
> > > InvalidateObsoleteReplicationSlot is called and we hold a SHARED lock
> > > on 'ReplicationSlotControlLock'. We are now holding the function using
> > > the sleep
> > > if (cause == RS_INVAL_WAL_LEVEL)
> > > {
> > > while (bsleep)
> > > sleep(1);
> > > }
> > >
> > > Now we create a copy of the slot since 'copy_replication_slot' takes
> > > a SHARED lock on 'ReplicationSlotControlLock'. It will take the lock
> > > and fetch the info of the source slot (the slot is not invalidated
> > > till now). and the function 'copy_replication_slot' calls function
> > > 'create_logical_replication_slot' which takes a EXCLUSIVE lock on
> > > ReplicationSlotControlLock and hence it will wait for function
> > > InvalidateObsoleteReplicationSlot to release lock. Once the function
> > > 'InvalidateObsoleteReplicationSlot' releases the lock, the execution
> > > of 'create_logical_replication_slot' continues and creates a copy of
> > > the source slot.
> > >
> > > Now with the patch, 'copy_replication_slot' will take an EXCLUSIVE
> > > lock on 'ReplicationSlotControlLock'. to fetch the slot info. Hence,
> > > it will wait for the 'InvalidateObsoleteReplicationSlot' to release
> > > the lock and then fetch the source slot info and try to create the
> > > copied slot (which will fail as source slot is invalidated before we
> > > fetch its info)
> > >
> > > > Besides, do we need one more invalidated check in the following codes after
> > > > creating the slot ?
> > > >
> > > > /*
> > > > * Check if the source slot still exists and is valid. We regard it as
> > > > * invalid if the type of replication slot or name has been changed,
> > > > * or the restart_lsn either is invalid or has gone backward. (The
> > > > ...
> > > >
> > >
> > > This approach seems more feasible to me. It also resolves the issue
> > > suggested by Vignesh in [1]. I have made changes for the same in v3
> > > patch.
> > >
> >
> > I agree to check if the source slot got invalidated during the copy.
> > But why do we need to search the slot by the slot name again as
> > follows?
> >
> > + /* Check if source slot was invalidated while copying of slot */
> > + LWLockAcquire(ReplicationSlotControlLock, LW_SHARED);
> > +
> > + for (int i = 0; i < max_replication_slots; i++)
> > + {
> > + ReplicationSlot *s = &ReplicationSlotCtl->replication_slots[i];
> > +
> > + if (s->in_use && strcmp(NameStr(s->data.name),
> > NameStr(*src_name)) == 0)
> > + {
> > + /* Copy the slot contents while holding spinlock */
> > + SpinLockAcquire(&s->mutex);
> > + first_slot_contents = *s;
> > + SpinLockRelease(&s->mutex);
> > + src = s;
> > + break;
> > + }
> > + }
> > +
> > + LWLockRelease(ReplicationSlotControlLock);
> >
> > I think 'src' already points to the source slot.
> >
> Hi Sawada san,
>
> Thanks for reviewing the patch.
> I have used the 'src' instead of iterating again. I have attached the
> updated v4 patch.
Thank you for updating the patch! I have one comment:
+ /* Check if source slot was invalidated while copying of slot */
+ SpinLockAcquire(&src->mutex);
+ first_slot_contents = *src;
+ SpinLockRelease(&src->mutex);
We don't need to copy the source slot contents again since we already
do as follows:
/* Copy data of source slot again */
SpinLockAcquire(&src->mutex);
second_slot_contents = *src;
SpinLockRelease(&src->mutex);
I think we can use second_slot_contents for that check.
I've investigated the slot invalidation and copying slots behaviors.
We cannot copy a slot if it doesn't reserve WAL, but IIUC the slot's
restart_lsn is not reset by slot invalidation due to other than
RS_INVAL_WAL_REMOVED. Therefore, it's possible that we copy a slot
invalidated by for example RS_INVAL_IDLE_TIMEOUT, and the copied
slot's restart_lsn might have already been removed, which ultimately
causes an assertion failure in ocpy_replication_slot():
#ifdef USE_ASSERT_CHECKING
/* Check that the restart_lsn is available */
{
XLogSegNo segno;
XLByteToSeg(copy_restart_lsn, segno, wal_segment_size);
Assert(XLogGetLastRemovedSegno() < segno);
}
#endif
I think this issue exists from v16 or later, I've not tested yet
though. If my understanding is right, this patch has to be
backpatched.
Regards,
--
Masahiko Sawada
Amazon Web Services: https://aws.amazon.com
^ permalink raw reply [nested|flat] 32+ messages in thread
* Re: Restrict copying of invalidated replication slots
@ 2025-02-23 01:16 Peter Smith <[email protected]>
parent: Shlok Kyal <[email protected]>
1 sibling, 1 reply; 32+ messages in thread
From: Peter Smith @ 2025-02-23 01:16 UTC (permalink / raw)
To: Shlok Kyal <[email protected]>; +Cc: Masahiko Sawada <[email protected]>; Zhijie Hou (Fujitsu) <[email protected]>; vignesh C <[email protected]>; PostgreSQL Hackers <[email protected]>
Some review comments for patch v2-0001.
======
Commit message
1.
Currently we can copy an invalidated logical and physical replication slot
using functions 'pg_copy_logical_replication_slot' and
'pg_copy_physical_replication_slot' respectively.
With this patch we will throw an error in such cases.
/we can copy an invalidated logical and physical replication slot/we
can copy invalidated logical and physical replication slots/
======
doc/src/sgml/func.sgml
pg_copy_physical_replication_slot:
2.
- is omitted, the same value as the
source slot is used.
+ is omitted, the same value as the source slot is used. Copy of an
+ invalidated physical replication slot in not allowed.
Typo /in/is/
Also, IMO you don't need to say "physical replication slot" because it
is clear from the function's name.
SUGGESTION
Copy of an invalidated slot is not allowed.
~~~
pg_copy_logical_replication_slot:
3.
+ Copy of an invalidated logical replication slot in not allowed.
Typo /in/is/
Also, IMO you don't need to say "logical replication slot" because it
is clear from the function's name.
SUGGESTION
Copy of an invalidated slot is not allowed.
======
src/backend/replication/slotfuncs.c
copy_replication_slot:
4.
+ /* Check if source slot was invalidated while copying of slot */
+ SpinLockAcquire(&src->mutex);
+ first_slot_contents = *src;
+ SpinLockRelease(&src->mutex);
+
+ src_isinvalidated = (first_slot_contents.data.invalidated != RS_INVAL_NONE);
+
+ if (src_isinvalidated)
+ ereport(ERROR,
+ (errmsg("could not copy replication slot \"%s\"",
+ NameStr(*src_name)),
+ errdetail("The source replication slot was invalidated during the
copy operation.")));
4a.
We already know that it was not invalid the FIRST time we looked at
it, so IMO we only need to confirm that the SECOND look gives the same
answer. IOW, I thought the code should be like below. (AFAICT
Sawada-san's review says the same as this).
Also, I think it is better to say "became invalidated" instead of "was
invalidated", to imply the state was known to be ok before the copy.
SUGGESTION
+ /* Check if source slot became invalidated during the copy operation */
+ if (second_slot_contents.data.invalidated != RS_INVAL_NONE)
+ ereport(ERROR,
~
4b.
Unnecessary parentheses in the ereport.
~
4c.
There seems some weird mix of tense "cannot copy" versus "could not
copy" already in this file. But, maybe at least you can be consistent
within the patch and always say "cannot".
======
Kind Regards,
Peter Smith.
Fujitsu Australia
^ permalink raw reply [nested|flat] 32+ messages in thread
* Re: Restrict copying of invalidated replication slots
@ 2025-02-24 11:07 Shlok Kyal <[email protected]>
parent: Peter Smith <[email protected]>
0 siblings, 0 replies; 32+ messages in thread
From: Shlok Kyal @ 2025-02-24 11:07 UTC (permalink / raw)
To: Peter Smith <[email protected]>; +Cc: Masahiko Sawada <[email protected]>; Zhijie Hou (Fujitsu) <[email protected]>; vignesh C <[email protected]>; PostgreSQL Hackers <[email protected]>
On Sun, 23 Feb 2025 at 06:46, Peter Smith <[email protected]> wrote:
>
> Some review comments for patch v2-0001.
>
> ======
> Commit message
>
> 1.
> Currently we can copy an invalidated logical and physical replication slot
> using functions 'pg_copy_logical_replication_slot' and
> 'pg_copy_physical_replication_slot' respectively.
> With this patch we will throw an error in such cases.
>
> /we can copy an invalidated logical and physical replication slot/we
> can copy invalidated logical and physical replication slots/
>
Updated the commit message
> ======
> doc/src/sgml/func.sgml
>
> pg_copy_physical_replication_slot:
>
> 2.
> - is omitted, the same value as the
> source slot is used.
> + is omitted, the same value as the source slot is used. Copy of an
> + invalidated physical replication slot in not allowed.
>
> Typo /in/is/
>
> Also, IMO you don't need to say "physical replication slot" because it
> is clear from the function's name.
>
> SUGGESTION
> Copy of an invalidated slot is not allowed.
>
Fixed
> ~~~
>
> pg_copy_logical_replication_slot:
>
> 3.
> + Copy of an invalidated logical replication slot in not allowed.
>
> Typo /in/is/
>
> Also, IMO you don't need to say "logical replication slot" because it
> is clear from the function's name.
>
> SUGGESTION
> Copy of an invalidated slot is not allowed.
>
>
Fixed
> ======
> src/backend/replication/slotfuncs.c
>
> copy_replication_slot:
>
> 4.
> + /* Check if source slot was invalidated while copying of slot */
> + SpinLockAcquire(&src->mutex);
> + first_slot_contents = *src;
> + SpinLockRelease(&src->mutex);
> +
> + src_isinvalidated = (first_slot_contents.data.invalidated != RS_INVAL_NONE);
> +
> + if (src_isinvalidated)
> + ereport(ERROR,
> + (errmsg("could not copy replication slot \"%s\"",
> + NameStr(*src_name)),
> + errdetail("The source replication slot was invalidated during the
> copy operation.")));
>
> 4a.
> We already know that it was not invalid the FIRST time we looked at
> it, so IMO we only need to confirm that the SECOND look gives the same
> answer. IOW, I thought the code should be like below. (AFAICT
> Sawada-san's review says the same as this).
>
> Also, I think it is better to say "became invalidated" instead of "was
> invalidated", to imply the state was known to be ok before the copy.
>
> SUGGESTION
>
> + /* Check if source slot became invalidated during the copy operation */
> + if (second_slot_contents.data.invalidated != RS_INVAL_NONE)
> + ereport(ERROR,
>
> ~
>
> 4b.
> Unnecessary parentheses in the ereport.
>
> ~
>
> 4c.
> There seems some weird mix of tense "cannot copy" versus "could not
> copy" already in this file. But, maybe at least you can be consistent
> within the patch and always say "cannot".
>
Fixed.
I have addressed the above comments in v5 patch [1].
[1]: https://www.postgresql.org/message-id/CANhcyEUHp6cRfaKf0ZqHCppCqpqzmsf5swpbnYGyRU%2BN%2Bihi%3DQ%40ma...
Thanks and Regards,
Shlok Kyal
^ permalink raw reply [nested|flat] 32+ messages in thread
end of thread, other threads:[~2025-02-24 11:07 UTC | newest]
Thread overview: 32+ messages (download: mbox mbox.gz follow: Atom feed)
-- links below jump to the message on this page --
2020-06-30 12:09 [PATCH v1] Replace min_safe_lsn with "distance" in pg_replication_slots Kyotaro Horiguchi <[email protected]>
2020-06-30 12:09 [PATCH v1] Replace min_safe_lsn with "distance" in pg_replication_slots Kyotaro Horiguchi <[email protected]>
2020-06-30 12:09 [PATCH v1] Replace min_safe_lsn with "distance" in pg_replication_slots Kyotaro Horiguchi <[email protected]>
2020-06-30 12:09 [PATCH v1] Replace min_safe_lsn with "distance" in pg_replication_slots Kyotaro Horiguchi <[email protected]>
2020-06-30 12:09 [PATCH v1] Replace min_safe_lsn with "distance" in pg_replication_slots Kyotaro Horiguchi <[email protected]>
2020-06-30 12:09 [PATCH v1] Replace min_safe_lsn with "distance" in pg_replication_slots Kyotaro Horiguchi <[email protected]>
2020-06-30 12:09 [PATCH v1] Replace min_safe_lsn with "distance" in pg_replication_slots Kyotaro Horiguchi <[email protected]>
2020-06-30 12:09 [PATCH v1] Replace min_safe_lsn with "distance" in pg_replication_slots Kyotaro Horiguchi <[email protected]>
2020-06-30 12:09 [PATCH v1] Replace min_safe_lsn with "distance" in pg_replication_slots Kyotaro Horiguchi <[email protected]>
2020-06-30 12:09 [PATCH v1] Replace min_safe_lsn with "distance" in pg_replication_slots Kyotaro Horiguchi <[email protected]>
2020-06-30 12:09 [PATCH v1] Replace min_safe_lsn with "distance" in pg_replication_slots Kyotaro Horiguchi <[email protected]>
2020-06-30 12:09 [PATCH v1] Replace min_safe_lsn with "distance" in pg_replication_slots Kyotaro Horiguchi <[email protected]>
2020-06-30 12:09 [PATCH v1] Replace min_safe_lsn with "distance" in pg_replication_slots Kyotaro Horiguchi <[email protected]>
2020-06-30 12:09 [PATCH v1] Replace min_safe_lsn with "distance" in pg_replication_slots Kyotaro Horiguchi <[email protected]>
2020-06-30 12:09 [PATCH v1] Replace min_safe_lsn with "distance" in pg_replication_slots Kyotaro Horiguchi <[email protected]>
2020-06-30 12:09 [PATCH v1] Replace min_safe_lsn with "distance" in pg_replication_slots Kyotaro Horiguchi <[email protected]>
2020-06-30 12:09 [PATCH v1] Replace min_safe_lsn with "distance" in pg_replication_slots Kyotaro Horiguchi <[email protected]>
2020-06-30 12:09 [PATCH v1] Replace min_safe_lsn with "distance" in pg_replication_slots Kyotaro Horiguchi <[email protected]>
2020-06-30 12:09 [PATCH v1] Replace min_safe_lsn with "distance" in pg_replication_slots Kyotaro Horiguchi <[email protected]>
2020-06-30 12:09 [PATCH v1] Replace min_safe_lsn with "distance" in pg_replication_slots Kyotaro Horiguchi <[email protected]>
2025-02-04 09:56 Restrict copying of invalidated replication slots Shlok Kyal <[email protected]>
2025-02-13 10:23 ` Re: Restrict copying of invalidated replication slots vignesh C <[email protected]>
2025-02-17 11:31 ` Re: Restrict copying of invalidated replication slots Shlok Kyal <[email protected]>
2025-02-18 09:55 ` RE: Restrict copying of invalidated replication slots Zhijie Hou (Fujitsu) <[email protected]>
2025-02-19 11:46 ` Re: Restrict copying of invalidated replication slots Shlok Kyal <[email protected]>
2025-02-20 19:43 ` Re: Restrict copying of invalidated replication slots Masahiko Sawada <[email protected]>
2025-02-21 12:30 ` Re: Restrict copying of invalidated replication slots Shlok Kyal <[email protected]>
2025-02-21 23:18 ` Re: Restrict copying of invalidated replication slots Masahiko Sawada <[email protected]>
2025-02-23 01:16 ` Re: Restrict copying of invalidated replication slots Peter Smith <[email protected]>
2025-02-24 11:07 ` Re: Restrict copying of invalidated replication slots Shlok Kyal <[email protected]>
2025-02-17 05:07 ` Re: Restrict copying of invalidated replication slots Peter Smith <[email protected]>
2025-02-17 11:34 ` Re: Restrict copying of invalidated replication slots Shlok Kyal <[email protected]>
This inbox is served by agora; see mirroring instructions
for how to clone and mirror all data and code used for this inbox