public inbox for [email protected]  

help / color / mirror / Atom feed

[PATCH v20 2/3] Add WAL relief vent for replication slots
125+ messages / 18 participants
[nested] [flat]

* [PATCH v20 2/3] Add WAL relief vent for replication slots
@ 2017-12-21 12:20 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 125+ messages in thread

From: Kyotaro Horiguchi @ 2017-12-21 12:20 UTC (permalink / raw)

Replication slot is useful to maintain replication connection in the
configurations where replication is so delayed that connection is
broken. On the other hand so many WAL files can fill up disk that the
master downs by a long delay. This feature, which is activated by a
GUC "max_slot_wal_keep_size", protects master servers from suffering
disk full by limiting the number of WAL files reserved by replication
slots.
---
 contrib/test_decoding/expected/ddl.out        |   4 +-
 contrib/test_decoding/sql/ddl.sql             |   2 +
 doc/src/sgml/catalogs.sgml                    |  48 +++
 doc/src/sgml/config.sgml                      |  23 ++
 doc/src/sgml/high-availability.sgml           |   8 +-
 src/backend/access/transam/xlog.c             | 361 ++++++++++++++++--
 src/backend/catalog/system_views.sql          |   4 +-
 src/backend/replication/slot.c                |   1 +
 src/backend/replication/slotfuncs.c           |  39 +-
 src/backend/utils/misc/guc.c                  |  13 +
 src/backend/utils/misc/postgresql.conf.sample |   1 +
 src/include/access/xlog.h                     |  19 +
 src/include/catalog/pg_proc.dat               |   6 +-
 src/test/recovery/t/018_replslot_limit.pl     | 202 ++++++++++
 src/test/regress/expected/rules.out           |   6 +-
 15 files changed, 699 insertions(+), 38 deletions(-)
 create mode 100644 src/test/recovery/t/018_replslot_limit.pl

diff --git a/contrib/test_decoding/expected/ddl.out b/contrib/test_decoding/expected/ddl.out
index 2c999fd3eb..cf0318f697 100644
--- a/contrib/test_decoding/expected/ddl.out
+++ b/contrib/test_decoding/expected/ddl.out
@@ -723,8 +723,8 @@ SELECT pg_drop_replication_slot('regression_slot');
 (1 row)
 
 /* check that the slot is gone */
+\x
 SELECT * FROM pg_replication_slots;
- slot_name | plugin | slot_type | datoid | database | temporary | active | active_pid | xmin | catalog_xmin | restart_lsn | confirmed_flush_lsn 
------------+--------+-----------+--------+----------+-----------+--------+------------+------+--------------+-------------+---------------------
 (0 rows)
 
+\x
diff --git a/contrib/test_decoding/sql/ddl.sql b/contrib/test_decoding/sql/ddl.sql
index 856495c952..0f2b9992f7 100644
--- a/contrib/test_decoding/sql/ddl.sql
+++ b/contrib/test_decoding/sql/ddl.sql
@@ -387,4 +387,6 @@ SELECT data FROM pg_logical_slot_get_changes('regression_slot', NULL, NULL, 'inc
 SELECT pg_drop_replication_slot('regression_slot');
 
 /* check that the slot is gone */
+\x
 SELECT * FROM pg_replication_slots;
+\x
diff --git a/doc/src/sgml/catalogs.sgml b/doc/src/sgml/catalogs.sgml
index 64614b569c..01a7802ed4 100644
--- a/doc/src/sgml/catalogs.sgml
+++ b/doc/src/sgml/catalogs.sgml
@@ -9907,6 +9907,54 @@ SELECT * FROM pg_locks pl LEFT JOIN pg_prepared_xacts ppx
       </entry>
      </row>
 
+     <row>
+      <entry><structfield>wal_status</structfield></entry>
+      <entry><type>text</type></entry>
+      <entry></entry>
+
+      <entry>Availability of WAL files claimed by this slot.
+      Valid values are:
+       <simplelist>
+        <member>
+         <literal>normal</literal> means that the claimed files
+         are within <varname>max_wal_size</varname>
+        </member>
+        <member>
+         <literal>keeping</literal> means that <varname>max_wal_size</varname>
+         is exceeded but still held by replication slots or
+         <varname>wal_keep_segments</varname>
+        </member>
+        <member>
+         <literal>losing</literal> means that some of the files are on the verge
+         of deletion, but can still be accessed by a session that's currently
+         reading it
+        </member>
+        <member>
+         <literal>lost</literal> means that some of them are definitely lost
+         and the session using this slot cannot continue replication.
+         This state also implies that the session using this slot has been
+         stopped.
+        </member>
+       </simplelist>
+      The last two states are seen only when
+      <xref linkend="guc-max-slot-wal-keep-size"/> is
+      non-negative. If <structfield>restart_lsn</structfield> is NULL, this
+      field is null.
+      </entry>
+     </row>
+
+     <row>
+      <entry><structfield>remain</structfield></entry>
+      <entry><type>bigint</type></entry>
+      <entry></entry>
+      <entry>The amount in bytes of WAL that can be written before this slot
+        loses required WAL files.
+        If <structfield>restart_lsn</structfield> is null or
+        <structfield>wal_status</structfield> is <literal>losing</literal>
+        or <literal>lost</literal>, this field is null.
+      </entry>
+     </row>
+
     </tbody>
    </tgroup>
   </table>
diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index 2de21903a1..dc99c6868a 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -3758,6 +3758,29 @@ restore_command = 'copy "C:\\server\\archivedir\\%f" "%p"'  # Windows
       </listitem>
      </varlistentry>
 
+      <varlistentry id="guc-max-slot-wal-keep-size" xreflabel="max_slot_wal_keep_size">
+       <term><varname>max_slot_wal_keep_size</varname> (<type>integer</type>)
+       <indexterm>
+        <primary><varname>max_slot_wal_keep_size</varname> configuration parameter</primary>
+       </indexterm>
+       </term>
+       <listitem>
+       <para>
+        Specify the maximum size of WAL files
+        that <link linkend="streaming-replication-slots">replication
+        slots</link> are allowed to retain in the <filename>pg_wal</filename>
+        directory at checkpoint time.
+        If <varname>max_slot_wal_keep_size</varname> is -1 (the default),
+        replication slots retain unlimited amount of WAL files.  If
+        restart_lsn of a replication slot gets behind more than that megabytes
+        from the current LSN, the standby using the slot may no longer be able
+        to continue replication due to removal of required WAL files. You
+        can see the WAL availability of replication slots
+        in <link linkend="view-pg-replication-slots">pg_replication_slots</link>.
+       </para>
+       </listitem>
+      </varlistentry>
+
      <varlistentry id="guc-wal-sender-timeout" xreflabel="wal_sender_timeout">
       <term><varname>wal_sender_timeout</varname> (<type>integer</type>)
       <indexterm>
diff --git a/doc/src/sgml/high-availability.sgml b/doc/src/sgml/high-availability.sgml
index b5d32bb720..624e5f94ad 100644
--- a/doc/src/sgml/high-availability.sgml
+++ b/doc/src/sgml/high-availability.sgml
@@ -925,9 +925,11 @@ primary_conninfo = 'host=192.168.1.50 port=5432 user=foo password=foopass'
     <xref linkend="guc-archive-command"/>.
     However, these methods often result in retaining more WAL segments than
     required, whereas replication slots retain only the number of segments
-    known to be needed.  An advantage of these methods is that they bound
-    the space requirement for <literal>pg_wal</literal>; there is currently no way
-    to do this using replication slots.
+    known to be needed.  On the other hand, replication slots can retain so
+    many WAL segments that they fill up the space allocated
+    for <literal>pg_wal</literal>;
+    <xref linkend="guc-max-slot-wal-keep-size"/> limits the size of WAL files
+    retained by replication slots.
    </para>
    <para>
     Similarly, <xref linkend="guc-hot-standby-feedback"/>
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 1951103b26..e68a89fded 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -105,6 +105,7 @@ int			wal_level = WAL_LEVEL_MINIMAL;
 int			CommitDelay = 0;	/* precommit delay in microseconds */
 int			CommitSiblings = 5; /* # concurrent xacts needed to sleep */
 int			wal_retrieve_retry_interval = 5000;
+int			max_slot_wal_keep_size_mb = -1;
 
 #ifdef WAL_DEBUG
 bool		XLOG_DEBUG = false;
@@ -758,7 +759,7 @@ static ControlFileData *ControlFile = NULL;
  */
 #define UsableBytesInPage (XLOG_BLCKSZ - SizeOfXLogShortPHD)
 
-/* Convert min_wal_size_mb and max_wal_size_mb to equivalent segment count */
+/* Convert values of GUCs measured in megabytes to equiv. segment count */
 #define ConvertToXSegs(x, segsize)	\
 	(x / ((segsize) / (1024 * 1024)))
 
@@ -895,6 +896,7 @@ static void checkTimeLineSwitch(XLogRecPtr lsn, TimeLineID newTLI,
 static void LocalSetXLogInsertAllowed(void);
 static void CreateEndOfRecoveryRecord(void);
 static void CheckPointGuts(XLogRecPtr checkPointRedo, int flags);
+static XLogSegNo GetOldestKeepSegment(XLogRecPtr currpos, XLogRecPtr minSlotPtr);
 static void KeepLogSeg(XLogRecPtr recptr, XLogSegNo *logSegNo);
 static XLogRecPtr XLogGetReplicationSlotMinimumLSN(void);
 
@@ -3929,8 +3931,56 @@ XLogGetLastRemovedSegno(void)
 }
 
 /*
- * Update the last removed segno pointer in shared memory, to reflect
- * that the given XLOG file has been removed.
+ * Scan the WAL directory then set the lastRemovedSegNo.
+ *
+ * In the case we need to know the last removed segment before the first
+ * checkpoint runs, call this function to initialize the variable by scanning
+ * the WAL directory.
+ */
+XLogSegNo
+ScanAndSetLastRemovedSegno(void)
+{
+	DIR		*xldir;
+	struct dirent *xlde;
+	XLogSegNo segno;
+
+	xldir = AllocateDir(XLOGDIR);
+	while ((xlde = ReadDir(xldir, XLOGDIR)) != NULL)
+	{
+		TimeLineID tli;
+		XLogSegNo fsegno;
+
+		/* Ignore files that are not XLOG segments */
+		if (!IsXLogFileName(xlde->d_name) &&
+			!IsPartialXLogFileName(xlde->d_name))
+			continue;
+
+		XLogFromFileName(xlde->d_name, &tli, &fsegno, wal_segment_size);
+
+		/*
+		 * Get minimum segment ignoring timeline ID, the same way with
+		 * RemoveOldXlogFiles().
+		 */
+		if (segno == 0 || fsegno < segno)
+			segno = fsegno;
+	}
+
+	FreeDir(xldir);
+
+	/* Update the last removed segno, not making retrogression. */
+	SpinLockAcquire(&XLogCtl->info_lck);
+	if (segno > XLogCtl->lastRemovedSegNo)
+		XLogCtl->lastRemovedSegNo = segno;
+	else
+		segno = XLogCtl->lastRemovedSegNo;
+	SpinLockRelease(&XLogCtl->info_lck);
+
+	return segno;
+}
+
+/*
+ * Update the last removed segno pointer in shared memory, to reflect that the
+ * given XLOG file has been removed.
  */
 static void
 UpdateLastRemovedPtr(char *filename)
@@ -9441,6 +9491,201 @@ CreateRestartPoint(int flags)
 	return true;
 }
 
+/*
+ * Detect availability of the record at given targetLSN.
+ *
+ * targetLSN is restart_lsn of a slot.
+ * walsender_pid is the slot's walsender PID.
+ *
+ * Returns one of the following enum values.
+ *
+ * WALAVAIL_NORMAL means targetLSN is available because it is in the range of
+ * max_wal_size.  If max_slot_wal_keep_size is smaller than max_wal_size, this
+ * state is not returned.
+ *
+ * WALAVAIL_PRESERVED means it is still available by preserving extra segments
+ * beyond max_wal_size.
+ *
+ * WALAVAIL_BEING_REMOVED means it is being removed or already removed but the
+ * replication stream on the given slot is live yet. The state may transit to
+ * WALAVAIL_PRESERVED or WALAVAIL_NORMAL state if the walsender advances
+ * restart_lsn.
+ *
+ * WALAVAIL_REMOVED means it is definitely lost. The replication stream on the
+ * slot cannot continue.
+ *
+ * returns WALAVAIL_NULL if restart_lsn is invalid.
+ */
+WalAvailability
+GetWalAvailability(XLogRecPtr restart_lsn, pid_t walsender_pid)
+{
+	XLogRecPtr currpos;
+	XLogRecPtr slotPtr;
+	XLogSegNo currSeg;		/* segid of currpos */
+	XLogSegNo restartSeg;	/* segid of restart_lsn */
+	XLogSegNo oldestSeg;	/* actual oldest segid */
+	XLogSegNo oldestSegMaxWalSize;	/* oldest segid kept by max_wal_size */
+	XLogSegNo oldestSlotSeg;/* oldest segid kept by slot */
+	uint64	  keepSegs;
+
+	/* the case where the slot has never been activated */
+	if (XLogRecPtrIsInvalid(restart_lsn))
+		return WALAVAIL_INVALID_LSN;
+
+	currpos = GetXLogWriteRecPtr();
+
+	/* calculate oldest segment currently needed by slots */
+	XLByteToSeg(restart_lsn, restartSeg, wal_segment_size);
+	slotPtr = XLogGetReplicationSlotMinimumLSN();
+	oldestSlotSeg = GetOldestKeepSegment(currpos, slotPtr);
+
+	/* find the oldest extant segment file */
+	oldestSeg = XLogGetLastRemovedSegno() + 1;
+
+	/* initialize last removed segno if not yet */
+	if (oldestSeg == 1)
+		oldestSeg = ScanAndSetLastRemovedSegno() + 1;
+
+	/* calculate oldest segment by max_wal_size */
+	XLByteToSeg(currpos, currSeg, wal_segment_size);
+	keepSegs = ConvertToXSegs(max_wal_size_mb, wal_segment_size) + 1;
+
+	if (currSeg > keepSegs)
+		oldestSegMaxWalSize = currSeg - keepSegs;
+	else
+		oldestSegMaxWalSize = 1;
+
+	/*
+	 * If max_slot_wal_keep_size has changed after the last call, the segment
+	 * that would been kept by the current setting might have been lost by the
+	 * previous setting. No point in showing normal or keeping status values if
+	 * the restartSeg is known to be lost.
+	 */
+	if (restartSeg >= oldestSeg)
+	{
+		/*
+		 * show "normal" when restartSeg is within max_wal_size. If
+		 * max_slot_wal_keep_size is smaller than max_wal_size, there's no
+		 * point in showing the status.
+		 */
+		if ((max_slot_wal_keep_size_mb <= 0 ||
+			 max_slot_wal_keep_size_mb >= max_wal_size_mb) &&
+			oldestSegMaxWalSize <= restartSeg)
+			return WALAVAIL_NORMAL;
+
+		/* being retained by slots */
+		if (oldestSlotSeg <= restartSeg)
+			return WALAVAIL_PRESERVED;
+	}
+	
+	/*
+	 * The segment is already lost or being lost. If the oldest segment is just
+	 * after the restartSeg, running walsender may be reading the just removed
+	 * segment. The walsender may safely move to the oldest existing segment in
+	 * that case.
+	 */
+	if (oldestSeg == restartSeg + 1 && walsender_pid != 0)
+		return WALAVAIL_BEING_REMOVED;
+
+	/* definitely lost. the walsender can no longer restart */
+	return WALAVAIL_REMOVED;
+}
+
+/*
+ * Returns minimum segment number that the next checkpoint must leave
+ * considering wal_keep_segments, replication slots and
+ * max_slot_wal_keep_size.
+ *
+ * currLSN is the current insert location.
+ * minSlotLSN is the minimum restart_lsn of all active slots.
+ */
+static XLogSegNo
+GetOldestKeepSegment(XLogRecPtr currLSN, XLogRecPtr minSlotLSN)
+{
+	XLogSegNo	currSeg;
+	XLogSegNo	minSlotSeg;
+	uint64		keepSegs = 0;	/* # of segments actually kept */
+
+	XLByteToSeg(currLSN, currSeg, wal_segment_size);
+	XLByteToSeg(minSlotLSN, minSlotSeg, wal_segment_size);
+
+	/*
+	 * Calculate how many segments are kept by slots first. The second
+	 * term of the condition is just a sanity check.
+	 */
+	if (minSlotLSN != InvalidXLogRecPtr && minSlotSeg <= currSeg)
+		keepSegs = currSeg - minSlotSeg;
+
+	/* Cap keepSegs by max_slot_wal_keep_size */
+	if (max_slot_wal_keep_size_mb >= 0)
+	{
+		uint64 limitSegs;
+
+		limitSegs = ConvertToXSegs(max_slot_wal_keep_size_mb, wal_segment_size);
+
+		/* Reduce it if slots already reserves too many. */
+		if (limitSegs < keepSegs)
+			keepSegs = limitSegs;
+	}
+
+	/* but, keep at least wal_keep_segments segments if any */
+	if (wal_keep_segments > 0 && keepSegs < wal_keep_segments)
+		keepSegs = wal_keep_segments;
+
+	/* avoid underflow, don't go below 1 */
+	if (currSeg <= keepSegs)
+		return 1;
+
+	return currSeg - keepSegs;
+}
+
+/*
+ * Calculate remaining bytes until WAL segment for targetLSN will be removed.
+ */
+int64
+DistanceToWalRemoval(XLogRecPtr currLSN, XLogRecPtr targetLSN)
+{
+	XLogSegNo	currSeg;
+	uint64		limitSegs = 0;
+	int64 		restbytes;
+	uint64		fragbytes;
+	XLogSegNo	targetSeg;
+
+	XLByteToSeg(currLSN, currSeg, wal_segment_size);
+
+	/* Calculate how far back WAL segments are preserved */
+	if (max_slot_wal_keep_size_mb >= 0)
+		limitSegs = ConvertToXSegs(max_slot_wal_keep_size_mb, wal_segment_size);
+
+	if (wal_keep_segments > 0 && limitSegs < wal_keep_segments)
+		limitSegs = wal_keep_segments;
+
+	XLByteToSeg(targetLSN, targetSeg, wal_segment_size);
+
+	/* avoid underflow */
+	if (targetSeg + limitSegs < currSeg)
+		return 0;
+
+	/*
+	 * This slot still has all required segments. Calculate how
+	 * many LSN bytes the slot has until it loses targetLSN.
+	 */
+	fragbytes = wal_segment_size - (currLSN % wal_segment_size);
+	XLogSegNoOffsetToRecPtr(targetSeg + limitSegs - currSeg,
+							fragbytes, wal_segment_size,
+							restbytes);
+
+	/*
+	 * Not realistic, but make sure that it is not out of the
+	 * range of int64. No problem to do so since such large values
+	 * have no significant difference.
+	 */
+	if (restbytes > PG_INT64_MAX)
+		restbytes = PG_INT64_MAX;
+
+	return restbytes;
+}
+
 /*
  * Retreat *logSegNo to the last segment that we need to retain because of
  * either wal_keep_segments or replication slots.
@@ -9452,38 +9697,102 @@ CreateRestartPoint(int flags)
 static void
 KeepLogSeg(XLogRecPtr recptr, XLogSegNo *logSegNo)
 {
-	XLogSegNo	segno;
-	XLogRecPtr	keep;
+	static XLogSegNo last_lost_segs = 0;
+	static int last_nslots = 0;
+	static char *last_slot_name = NULL;
+	XLogRecPtr	slotminptr = InvalidXLogRecPtr;
+	XLogSegNo	minSegNo;
+	XLogSegNo	minSlotSegNo;
+	int			nslots_affected = 0;
 
-	XLByteToSeg(recptr, segno, wal_segment_size);
-	keep = XLogGetReplicationSlotMinimumLSN();
+	if (max_replication_slots > 0)
+		slotminptr = XLogGetReplicationSlotMinimumLSN();
 
-	/* compute limit for wal_keep_segments first */
-	if (wal_keep_segments > 0)
+	/*
+	 * We should keep certain number of WAL segments after this checkpoint.
+	 */
+	minSegNo = GetOldestKeepSegment(recptr, slotminptr);
+
+	/*
+	 * Warn the checkpoint is going to flush the segments required by
+	 * replication slots.
+	 */
+	if (!XLogRecPtrIsInvalid(slotminptr))
 	{
-		/* avoid underflow, don't go below 1 */
-		if (segno <= wal_keep_segments)
-			segno = 1;
-		else
-			segno = segno - wal_keep_segments;
+		Assert (max_replication_slots > 0);
+
+		XLByteToSeg(slotminptr, minSlotSegNo, wal_segment_size);
+
+		if (minSlotSegNo < minSegNo)
+		{
+			/* Some slots has lost required segments */
+			XLogSegNo	lost_segs = minSegNo - minSlotSegNo;
+			ReplicationSlot *earliest = NULL;
+			char	   *earliest_name = NULL;
+			int			i;
+
+			/* Find the most affected slot */
+			LWLockAcquire(ReplicationSlotControlLock, LW_SHARED);
+			for (i = 0 ; i < max_replication_slots ; i++)
+			{
+				ReplicationSlot *s =
+					&ReplicationSlotCtl->replication_slots[i];
+				XLogSegNo slotSegNo;
+
+				XLByteToSeg(s->data.restart_lsn, slotSegNo, wal_segment_size);
+
+				if (s->in_use && s->active_pid == 0 && slotSegNo < minSegNo)
+				{
+					nslots_affected++;
+
+					if (earliest == NULL ||
+						s->data.restart_lsn < earliest->data.restart_lsn)
+						earliest = s;
+				}
+			}
+
+			if (earliest)
+			{
+				MemoryContext oldcxt = MemoryContextSwitchTo(TopMemoryContext);
+				earliest_name = pstrdup(NameStr(earliest->data.name));
+				MemoryContextSwitchTo(oldcxt);
+			}
+
+			LWLockRelease(ReplicationSlotControlLock);
+
+			/* Emit WARNING if something has changed */
+			if (earliest_name &&
+				(last_lost_segs != lost_segs || last_nslots != nslots_affected))
+			{
+				ereport(WARNING,
+						(errmsg_plural ("%d replication slot has lost required WAL segments by %lu segments",
+										"%d replication slots have lost required WAL segments by %lu segments",
+										nslots_affected, nslots_affected,
+										lost_segs),
+						 errdetail("Most affected slot is %s.",
+								   earliest_name)));
+
+				if (last_slot_name)
+					pfree(last_slot_name);
+				last_slot_name = earliest_name;
+				last_lost_segs = lost_segs;
+				last_nslots = nslots_affected;
+			}
+		}
 	}
 
-	/* then check whether slots limit removal further */
-	if (max_replication_slots > 0 && keep != InvalidXLogRecPtr)
+	/* Reset the state if no affected slots remain. */
+	if (nslots_affected == 0 && last_slot_name)
 	{
-		XLogSegNo	slotSegNo;
-
-		XLByteToSeg(keep, slotSegNo, wal_segment_size);
-
-		if (slotSegNo <= 0)
-			segno = 1;
-		else if (slotSegNo < segno)
-			segno = slotSegNo;
+		pfree(last_slot_name);
+		last_slot_name = NULL;
+		last_lost_segs = 0;
+		last_nslots = 0;
 	}
 
 	/* don't delete WAL segments newer than the calculated segment */
-	if (segno < *logSegNo)
-		*logSegNo = segno;
+	if (minSegNo < *logSegNo)
+		*logSegNo = minSegNo;
 }
 
 /*
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index 83d00c6cde..775b8b7f20 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -863,7 +863,9 @@ CREATE VIEW pg_replication_slots AS
             L.xmin,
             L.catalog_xmin,
             L.restart_lsn,
-            L.confirmed_flush_lsn
+            L.confirmed_flush_lsn,
+            L.wal_status,
+            L.remain
     FROM pg_get_replication_slots() AS L
             LEFT JOIN pg_database D ON (L.datoid = D.oid);
 
diff --git a/src/backend/replication/slot.c b/src/backend/replication/slot.c
index d90c7235e9..a26f7999aa 100644
--- a/src/backend/replication/slot.c
+++ b/src/backend/replication/slot.c
@@ -49,6 +49,7 @@
 #include "storage/proc.h"
 #include "storage/procarray.h"
 #include "utils/builtins.h"
+#include "utils/memutils.h"
 
 /*
  * Replication slot on-disk data structure.
diff --git a/src/backend/replication/slotfuncs.c b/src/backend/replication/slotfuncs.c
index ce0c9127bc..47cd4375a1 100644
--- a/src/backend/replication/slotfuncs.c
+++ b/src/backend/replication/slotfuncs.c
@@ -234,7 +234,7 @@ pg_drop_replication_slot(PG_FUNCTION_ARGS)
 Datum
 pg_get_replication_slots(PG_FUNCTION_ARGS)
 {
-#define PG_GET_REPLICATION_SLOTS_COLS 11
+#define PG_GET_REPLICATION_SLOTS_COLS 13
 	ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo;
 	TupleDesc	tupdesc;
 	Tuplestorestate *tupstore;
@@ -288,6 +288,7 @@ pg_get_replication_slots(PG_FUNCTION_ARGS)
 		Oid			database;
 		NameData	slot_name;
 		NameData	plugin;
+		WalAvailability walstate;
 		int			i;
 
 		if (!slot->in_use)
@@ -355,6 +356,42 @@ pg_get_replication_slots(PG_FUNCTION_ARGS)
 		else
 			nulls[i++] = true;
 
+		walstate = GetWalAvailability(restart_lsn, active_pid);
+
+		switch (walstate)
+		{
+			case WALAVAIL_INVALID_LSN:
+				nulls[i++] = true;
+				break;
+
+			case WALAVAIL_NORMAL:
+				values[i++] = CStringGetTextDatum("normal");
+				break;
+
+			case WALAVAIL_PRESERVED:
+				values[i++] = CStringGetTextDatum("keeping");
+				break;
+
+			case WALAVAIL_BEING_REMOVED:
+				values[i++] = CStringGetTextDatum("losing");
+				break;
+
+			case WALAVAIL_REMOVED:
+				values[i++] = CStringGetTextDatum("lost");
+				break;
+		}
+
+		if (max_slot_wal_keep_size_mb >= 0 &&
+			(walstate == WALAVAIL_NORMAL ||
+			 walstate == WALAVAIL_PRESERVED))
+		{
+			values[i++] =
+				Int64GetDatum(DistanceToWalRemoval(GetXLogWriteRecPtr(),
+												   restart_lsn));
+		}
+		else
+			nulls[i++] = true;
+
 		tuplestore_putvalues(tupstore, tupdesc, values, nulls);
 	}
 	LWLockRelease(ReplicationSlotControlLock);
diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c
index 79bc7ac8ca..54cd5f6420 100644
--- a/src/backend/utils/misc/guc.c
+++ b/src/backend/utils/misc/guc.c
@@ -2771,6 +2771,19 @@ static struct config_int ConfigureNamesInt[] =
 		NULL, NULL, NULL
 	},
 
+	{
+		{"max_slot_wal_keep_size", PGC_SIGHUP, REPLICATION_SENDING,
+			gettext_noop("Sets the maximum size of WAL space reserved by replication slots."),
+			gettext_noop("Replication slots will be marked as failed, and segments released "
+						 "for deletion or recycling, if this much space is occupied by WAL "
+						 "on disk."),
+			GUC_UNIT_MB
+		},
+		&max_slot_wal_keep_size_mb,
+		-1, -1, MAX_KILOBYTES,
+		NULL, NULL, NULL
+	},
+
 	{
 		{"wal_sender_timeout", PGC_USERSET, REPLICATION_SENDING,
 			gettext_noop("Sets the maximum time to wait for WAL replication."),
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index e9f8ca775d..0b696e7044 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -287,6 +287,7 @@
 #max_wal_senders = 10		# max number of walsender processes
 				# (change requires restart)
 #wal_keep_segments = 0		# in logfile segments; 0 disables
+#max_slot_wal_keep_size = -1	# measured in bytes; -1 disables
 #wal_sender_timeout = 60s	# in milliseconds; 0 disables
 
 #max_replication_slots = 10	# max number of replication slots
diff --git a/src/include/access/xlog.h b/src/include/access/xlog.h
index 9ec7b31cce..c1994eec6f 100644
--- a/src/include/access/xlog.h
+++ b/src/include/access/xlog.h
@@ -108,6 +108,7 @@ extern int	wal_segment_size;
 extern int	min_wal_size_mb;
 extern int	max_wal_size_mb;
 extern int	wal_keep_segments;
+extern int	max_slot_wal_keep_size_mb;
 extern int	XLOGbuffers;
 extern int	XLogArchiveTimeout;
 extern int	wal_retrieve_retry_interval;
@@ -255,6 +256,20 @@ typedef struct CheckpointStatsData
 
 extern CheckpointStatsData CheckpointStats;
 
+/*
+ * WAL segment availability status
+ *
+ * This is used as the return value of GetWalAvailability.
+ */
+typedef enum WalAvailability
+{
+	WALAVAIL_INVALID_LSN,			/* parameter error */
+	WALAVAIL_NORMAL,				/* WAL segment is within max_wal_size */
+	WALAVAIL_PRESERVED,				/* WAL segment is preserved by repslots */
+	WALAVAIL_BEING_REMOVED,			/* WAL segment is no longer preserved */
+	WALAVAIL_REMOVED				/* WAL segment has been removed */
+} WalAvailability;
+
 struct XLogRecData;
 
 extern XLogRecPtr XLogInsertRecord(struct XLogRecData *rdata,
@@ -268,6 +283,7 @@ extern int	XLogFileOpen(XLogSegNo segno);
 
 extern void CheckXLogRemoved(XLogSegNo segno, TimeLineID tli);
 extern XLogSegNo XLogGetLastRemovedSegno(void);
+extern XLogSegNo ScanAndSetLastRemovedSegno(void);
 extern void XLogSetAsyncXactLSN(XLogRecPtr record);
 extern void XLogSetReplicationSlotMinimumLSN(XLogRecPtr lsn);
 
@@ -305,6 +321,9 @@ extern void ShutdownXLOG(int code, Datum arg);
 extern void InitXLOGAccess(void);
 extern void CreateCheckPoint(int flags);
 extern bool CreateRestartPoint(int flags);
+extern WalAvailability GetWalAvailability(XLogRecPtr restart_lsn,
+										  pid_t walsender_pid);
+extern int64 DistanceToWalRemoval(XLogRecPtr currLSN, XLogRecPtr targetLSN);
 extern void XLogPutNextOid(Oid nextOid);
 extern XLogRecPtr XLogRestorePoint(const char *rpName);
 extern void UpdateFullPageWrites(void);
diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat
index a6a708cca9..2025f34bfd 100644
--- a/src/include/catalog/pg_proc.dat
+++ b/src/include/catalog/pg_proc.dat
@@ -9971,9 +9971,9 @@
   proname => 'pg_get_replication_slots', prorows => '10', proisstrict => 'f',
   proretset => 't', provolatile => 's', prorettype => 'record',
   proargtypes => '',
-  proallargtypes => '{name,name,text,oid,bool,bool,int4,xid,xid,pg_lsn,pg_lsn}',
-  proargmodes => '{o,o,o,o,o,o,o,o,o,o,o}',
-  proargnames => '{slot_name,plugin,slot_type,datoid,temporary,active,active_pid,xmin,catalog_xmin,restart_lsn,confirmed_flush_lsn}',
+  proallargtypes => '{name,name,text,oid,bool,bool,int4,xid,xid,pg_lsn,pg_lsn,text,int8}',
+  proargmodes => '{o,o,o,o,o,o,o,o,o,o,o,o,o}',
+  proargnames => '{slot_name,plugin,slot_type,datoid,temporary,active,active_pid,xmin,catalog_xmin,restart_lsn,confirmed_flush_lsn,wal_status,remain}',
   prosrc => 'pg_get_replication_slots' },
 { oid => '3786', descr => 'set up a logical replication slot',
   proname => 'pg_create_logical_replication_slot', provolatile => 'v',
diff --git a/src/test/recovery/t/018_replslot_limit.pl b/src/test/recovery/t/018_replslot_limit.pl
new file mode 100644
index 0000000000..84a1f3a9dd
--- /dev/null
+++ b/src/test/recovery/t/018_replslot_limit.pl
@@ -0,0 +1,202 @@
+# Test for replication slot limit
+# Ensure that max_slot_wal_keep_size limits the number of WAL files to
+# be kept by replication slots.
+
+use strict;
+use warnings;
+use File::Path qw(rmtree);
+use PostgresNode;
+use TestLib;
+use Test::More tests => 13;
+use Time::HiRes qw(usleep);
+
+$ENV{PGDATABASE} = 'postgres';
+
+# Initialize master node, setting wal-segsize to 1MB
+my $node_master = get_new_node('master');
+$node_master->init(allows_streaming => 1,
+				   extra => ['--wal-segsize=1'],
+				   min_wal_size => '2MB',
+				   max_wal_size => '4MB',
+				   log_checkpoints => 'yes');
+$node_master->start;
+$node_master->safe_psql('postgres', "SELECT pg_create_physical_replication_slot('rep1')");
+
+# The slot state and remain should be null before the first connection
+my $result = $node_master->safe_psql('postgres', "SELECT restart_lsn is NULL, wal_status is NULL, remain is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'");
+is($result, "t|t|t", 'check the state of non-reserved slot is "unknown"');
+
+
+# Take backup
+my $backup_name = 'my_backup';
+$node_master->backup($backup_name);
+
+# Create a standby linking to it using the replication slot
+my $node_standby = get_new_node('standby_1');
+$node_standby->init_from_backup($node_master, $backup_name, has_streaming => 1,
+								primary_slot_name => 'rep1');
+
+$node_standby->start;
+
+# Wait until standby has replayed enough data
+my $start_lsn = $node_master->lsn('write');
+$node_master->wait_for_catchup($node_standby, 'replay', $start_lsn);
+
+# Stop standby
+$node_standby->stop;
+
+
+# Preparation done, the slot is the state "normal" now
+$result = $node_master->safe_psql('postgres', "SELECT restart_lsn, wal_status, remain is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'");
+is($result, "$start_lsn|normal|t", 'check the catching-up state');
+
+# Advance WAL by five segments (= 5MB) on master
+advance_wal($node_master, 1);
+$node_master->safe_psql('postgres', "CHECKPOINT;");
+
+# The slot is always "safe" when fitting max_wal_size
+$result = $node_master->safe_psql('postgres', "SELECT restart_lsn, wal_status, remain is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'");
+is($result, "$start_lsn|normal|t", 'check that restart_lsn is in max_wal_size');
+
+advance_wal($node_master, 4);
+$node_master->safe_psql('postgres', "CHECKPOINT;");
+
+# The slot is always "safe" when max_slot_wal_keep_size is not set
+$result = $node_master->safe_psql('postgres', "SELECT restart_lsn, wal_status, remain is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'");
+is($result, "$start_lsn|normal|t", 'check that slot is working');
+
+# The standby can reconnect to master
+$node_standby->start;
+
+$start_lsn = $node_master->lsn('write');
+$node_master->wait_for_catchup($node_standby, 'replay', $start_lsn);
+
+$node_standby->stop;
+
+# Set max_slot_wal_keep_size on master
+my $max_slot_wal_keep_size_mb = 6;
+$node_master->append_conf('postgresql.conf', qq(
+max_slot_wal_keep_size = ${max_slot_wal_keep_size_mb}MB
+));
+$node_master->reload;
+
+# The slot is in safe state. The remaining bytes should be as almost
+# (max_slot_wal_keep_size + 1) times large as the segment size
+$result = $node_master->safe_psql('postgres', "SELECT restart_lsn, wal_status, pg_size_pretty(remain) as remain FROM pg_replication_slots WHERE slot_name = 'rep1'");
+is($result, "$start_lsn|normal|7168 kB", 'check that max_slot_wal_keep_size is working');
+
+# Advance WAL again then checkpoint, reducing remain by 2 MB.
+advance_wal($node_master, 2);
+$node_master->safe_psql('postgres', "CHECKPOINT;");
+
+# The slot is still working
+$result = $node_master->safe_psql('postgres', "SELECT restart_lsn, wal_status, pg_size_pretty(remain) as remain FROM pg_replication_slots WHERE slot_name = 'rep1'");
+is($result, "$start_lsn|normal|5120 kB", 'check that remaining byte is calculated correctly');
+
+# wal_keep_segments overrides max_slot_wal_keep_size
+$result = $node_master->safe_psql('postgres', "ALTER SYSTEM SET wal_keep_segments to 8; SELECT pg_reload_conf();");
+$result = $node_master->safe_psql('postgres', "SELECT restart_lsn, wal_status, pg_size_pretty(remain) as remain FROM pg_replication_slots WHERE slot_name = 'rep1'");
+is($result, "$start_lsn|normal|7168 kB", 'check that wal_keep_segments overrides max_slot_wal_keep_size');
+
+# restore wal_keep_segments
+$result = $node_master->safe_psql('postgres', "ALTER SYSTEM SET wal_keep_segments to 0; SELECT pg_reload_conf();");
+
+# Advance WAL again without checkpoint, reducing remain by 4 MB.
+advance_wal($node_master, 4);
+
+# Slot gets into 'keeping' state
+$result = $node_master->safe_psql('postgres', "SELECT restart_lsn, wal_status, pg_size_pretty(remain) as remain FROM pg_replication_slots WHERE slot_name = 'rep1'");
+is($result, "$start_lsn|keeping|1024 kB", 'check that the slot state changes to "keeping"');
+
+# do checkpoint so that the next checkpoint runs too early
+$node_master->safe_psql('postgres', "CHECKPOINT;");
+
+# Advance WAL again without checkpoint; remain goes to 0.
+advance_wal($node_master, 1);
+
+# Slot gets into 'lost' state
+$result = $node_master->safe_psql('postgres', "SELECT restart_lsn, wal_status, remain is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'");
+is($result, "$start_lsn|lost|t", 'check that the slot state changes to "lost"');
+
+# The standby still can connect to master before a checkpoint
+$node_standby->start;
+
+$start_lsn = $node_master->lsn('write');
+$node_master->wait_for_catchup($node_standby, 'replay', $start_lsn);
+
+$node_standby->stop;
+
+ok(!find_in_log($node_standby,
+				"requested WAL segment [0-9A-F]+ has already been removed"),
+   'check that required WAL segments are still available');
+
+# Advance WAL again, the slot loses the oldest segment.
+my $logstart = get_log_size($node_master);
+advance_wal($node_master, 7);
+$node_master->safe_psql('postgres', "CHECKPOINT;");
+
+# WARNING should be issued
+ok(find_in_log($node_master,
+			   "1 replication slot has lost required WAL segments by 1 segments\n".
+			   ".*Most affected slot is rep1.",
+			   $logstart),
+   'check that the warning is logged');
+
+# This slot should be broken
+$result = $node_master->safe_psql('postgres', "SELECT restart_lsn, wal_status, remain is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'");
+is($result, "$start_lsn|lost|t", 'check that the slot state changes to "lost"');
+
+# The standby no longer can connect to the master
+$logstart = get_log_size($node_standby);
+$node_standby->start;
+
+my $failed = 0;
+for (my $i = 0 ; $i < 10000 ; $i++)
+{
+	if (find_in_log($node_standby,
+					"requested WAL segment [0-9A-F]+ has already been removed",
+					$logstart))
+	{
+		$failed = 1;
+		last;
+	}
+	usleep(100_000);
+}
+ok($failed, 'check that replication has been broken');
+
+$node_standby->stop;
+
+#####################################
+# Advance WAL of $node by $n segments
+sub advance_wal
+{
+	my ($node, $n) = @_;
+
+	# Advance by $n segments (= (16 * $n) MB) on master
+	for (my $i = 0 ; $i < $n ; $i++)
+	{
+		$node->safe_psql('postgres', "CREATE TABLE t (); DROP TABLE t; SELECT pg_switch_wal();");
+	}
+}
+
+# return the size of logfile of $node in bytes
+sub get_log_size
+{
+	my ($node) = @_;
+
+	return (stat $node->logfile)[7];
+}
+
+# find $pat in logfile of $node after $off-th byte
+sub find_in_log
+{
+	my ($node, $pat, $off) = @_;
+
+	$off = 0 unless defined $off;
+	my $log = TestLib::slurp_file($node->logfile);
+	return 0 if (length($log) <= $off);
+
+	$log = substr($log, $off);
+
+	return $log =~ m/$pat/;
+}
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index 7245b0e13b..8688f7138f 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1462,8 +1462,10 @@ pg_replication_slots| SELECT l.slot_name,
     l.xmin,
     l.catalog_xmin,
     l.restart_lsn,
-    l.confirmed_flush_lsn
-   FROM (pg_get_replication_slots() l(slot_name, plugin, slot_type, datoid, temporary, active, active_pid, xmin, catalog_xmin, restart_lsn, confirmed_flush_lsn)
+    l.confirmed_flush_lsn,
+    l.wal_status,
+    l.remain
+   FROM (pg_get_replication_slots() l(slot_name, plugin, slot_type, datoid, temporary, active, active_pid, xmin, catalog_xmin, restart_lsn, confirmed_flush_lsn, wal_status, remain)
      LEFT JOIN pg_database d ON ((l.datoid = d.oid)));
 pg_roles| SELECT pg_authid.rolname,
     pg_authid.rolsuper,
-- 
2.18.2


----Next_Part(Tue_Mar_31_14_20_16_2020_885)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v20-0003-Allow-init-and-init_from_backup-to-set-arbitrary.patch"



^ permalink  raw  reply  [nested|flat] 125+ messages in thread

* [PATCH v19 2/3] Add WAL relief vent for replication slots
@ 2017-12-21 12:20 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 125+ messages in thread

From: Kyotaro Horiguchi @ 2017-12-21 12:20 UTC (permalink / raw)

Replication slot is useful to maintain replication connection in the
configurations where replication is so delayed that connection is
broken. On the other hand so many WAL files can fill up disk that the
master downs by a long delay. This feature, which is activated by a
GUC "max_slot_wal_keep_size", protects master servers from suffering
disk full by limiting the number of WAL files reserved by replication
slots.
---
 contrib/test_decoding/expected/ddl.out        |   4 +-
 contrib/test_decoding/sql/ddl.sql             |   2 +
 doc/src/sgml/catalogs.sgml                    |  48 +++
 doc/src/sgml/config.sgml                      |  23 ++
 doc/src/sgml/high-availability.sgml           |   8 +-
 src/backend/access/transam/xlog.c             | 354 ++++++++++++++++--
 src/backend/catalog/system_views.sql          |   4 +-
 src/backend/replication/slot.c                |   1 +
 src/backend/replication/slotfuncs.c           |  39 +-
 src/backend/utils/misc/guc.c                  |  13 +
 src/backend/utils/misc/postgresql.conf.sample |   1 +
 src/include/access/xlog.h                     |  19 +
 src/include/catalog/pg_proc.dat               |   6 +-
 src/test/recovery/t/018_replslot_limit.pl     | 202 ++++++++++
 src/test/regress/expected/rules.out           |   6 +-
 15 files changed, 694 insertions(+), 36 deletions(-)
 create mode 100644 src/test/recovery/t/018_replslot_limit.pl

diff --git a/contrib/test_decoding/expected/ddl.out b/contrib/test_decoding/expected/ddl.out
index 2c999fd3eb..cf0318f697 100644
--- a/contrib/test_decoding/expected/ddl.out
+++ b/contrib/test_decoding/expected/ddl.out
@@ -723,8 +723,8 @@ SELECT pg_drop_replication_slot('regression_slot');
 (1 row)
 
 /* check that the slot is gone */
+\x
 SELECT * FROM pg_replication_slots;
- slot_name | plugin | slot_type | datoid | database | temporary | active | active_pid | xmin | catalog_xmin | restart_lsn | confirmed_flush_lsn 
------------+--------+-----------+--------+----------+-----------+--------+------------+------+--------------+-------------+---------------------
 (0 rows)
 
+\x
diff --git a/contrib/test_decoding/sql/ddl.sql b/contrib/test_decoding/sql/ddl.sql
index 856495c952..0f2b9992f7 100644
--- a/contrib/test_decoding/sql/ddl.sql
+++ b/contrib/test_decoding/sql/ddl.sql
@@ -387,4 +387,6 @@ SELECT data FROM pg_logical_slot_get_changes('regression_slot', NULL, NULL, 'inc
 SELECT pg_drop_replication_slot('regression_slot');
 
 /* check that the slot is gone */
+\x
 SELECT * FROM pg_replication_slots;
+\x
diff --git a/doc/src/sgml/catalogs.sgml b/doc/src/sgml/catalogs.sgml
index 64614b569c..01a7802ed4 100644
--- a/doc/src/sgml/catalogs.sgml
+++ b/doc/src/sgml/catalogs.sgml
@@ -9907,6 +9907,54 @@ SELECT * FROM pg_locks pl LEFT JOIN pg_prepared_xacts ppx
       </entry>
      </row>
 
+     <row>
+      <entry><structfield>wal_status</structfield></entry>
+      <entry><type>text</type></entry>
+      <entry></entry>
+
+      <entry>Availability of WAL files claimed by this slot.
+      Valid values are:
+       <simplelist>
+        <member>
+         <literal>normal</literal> means that the claimed files
+         are within <varname>max_wal_size</varname>
+        </member>
+        <member>
+         <literal>keeping</literal> means that <varname>max_wal_size</varname>
+         is exceeded but still held by replication slots or
+         <varname>wal_keep_segments</varname>
+        </member>
+        <member>
+         <literal>losing</literal> means that some of the files are on the verge
+         of deletion, but can still be accessed by a session that's currently
+         reading it
+        </member>
+        <member>
+         <literal>lost</literal> means that some of them are definitely lost
+         and the session using this slot cannot continue replication.
+         This state also implies that the session using this slot has been
+         stopped.
+        </member>
+       </simplelist>
+      The last two states are seen only when
+      <xref linkend="guc-max-slot-wal-keep-size"/> is
+      non-negative. If <structfield>restart_lsn</structfield> is NULL, this
+      field is null.
+      </entry>
+     </row>
+
+     <row>
+      <entry><structfield>remain</structfield></entry>
+      <entry><type>bigint</type></entry>
+      <entry></entry>
+      <entry>The amount in bytes of WAL that can be written before this slot
+        loses required WAL files.
+        If <structfield>restart_lsn</structfield> is null or
+        <structfield>wal_status</structfield> is <literal>losing</literal>
+        or <literal>lost</literal>, this field is null.
+      </entry>
+     </row>
+
     </tbody>
    </tgroup>
   </table>
diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index 2de21903a1..dc99c6868a 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -3758,6 +3758,29 @@ restore_command = 'copy "C:\\server\\archivedir\\%f" "%p"'  # Windows
       </listitem>
      </varlistentry>
 
+      <varlistentry id="guc-max-slot-wal-keep-size" xreflabel="max_slot_wal_keep_size">
+       <term><varname>max_slot_wal_keep_size</varname> (<type>integer</type>)
+       <indexterm>
+        <primary><varname>max_slot_wal_keep_size</varname> configuration parameter</primary>
+       </indexterm>
+       </term>
+       <listitem>
+       <para>
+        Specify the maximum size of WAL files
+        that <link linkend="streaming-replication-slots">replication
+        slots</link> are allowed to retain in the <filename>pg_wal</filename>
+        directory at checkpoint time.
+        If <varname>max_slot_wal_keep_size</varname> is -1 (the default),
+        replication slots retain unlimited amount of WAL files.  If
+        restart_lsn of a replication slot gets behind more than that megabytes
+        from the current LSN, the standby using the slot may no longer be able
+        to continue replication due to removal of required WAL files. You
+        can see the WAL availability of replication slots
+        in <link linkend="view-pg-replication-slots">pg_replication_slots</link>.
+       </para>
+       </listitem>
+      </varlistentry>
+
      <varlistentry id="guc-wal-sender-timeout" xreflabel="wal_sender_timeout">
       <term><varname>wal_sender_timeout</varname> (<type>integer</type>)
       <indexterm>
diff --git a/doc/src/sgml/high-availability.sgml b/doc/src/sgml/high-availability.sgml
index b5d32bb720..624e5f94ad 100644
--- a/doc/src/sgml/high-availability.sgml
+++ b/doc/src/sgml/high-availability.sgml
@@ -925,9 +925,11 @@ primary_conninfo = 'host=192.168.1.50 port=5432 user=foo password=foopass'
     <xref linkend="guc-archive-command"/>.
     However, these methods often result in retaining more WAL segments than
     required, whereas replication slots retain only the number of segments
-    known to be needed.  An advantage of these methods is that they bound
-    the space requirement for <literal>pg_wal</literal>; there is currently no way
-    to do this using replication slots.
+    known to be needed.  On the other hand, replication slots can retain so
+    many WAL segments that they fill up the space allocated
+    for <literal>pg_wal</literal>;
+    <xref linkend="guc-max-slot-wal-keep-size"/> limits the size of WAL files
+    retained by replication slots.
    </para>
    <para>
     Similarly, <xref linkend="guc-hot-standby-feedback"/>
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 1951103b26..be9f218e7c 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -105,6 +105,7 @@ int			wal_level = WAL_LEVEL_MINIMAL;
 int			CommitDelay = 0;	/* precommit delay in microseconds */
 int			CommitSiblings = 5; /* # concurrent xacts needed to sleep */
 int			wal_retrieve_retry_interval = 5000;
+int			max_slot_wal_keep_size_mb = -1;
 
 #ifdef WAL_DEBUG
 bool		XLOG_DEBUG = false;
@@ -758,7 +759,7 @@ static ControlFileData *ControlFile = NULL;
  */
 #define UsableBytesInPage (XLOG_BLCKSZ - SizeOfXLogShortPHD)
 
-/* Convert min_wal_size_mb and max_wal_size_mb to equivalent segment count */
+/* Convert values of GUCs measured in megabytes to equiv. segment count */
 #define ConvertToXSegs(x, segsize)	\
 	(x / ((segsize) / (1024 * 1024)))
 
@@ -895,6 +896,7 @@ static void checkTimeLineSwitch(XLogRecPtr lsn, TimeLineID newTLI,
 static void LocalSetXLogInsertAllowed(void);
 static void CreateEndOfRecoveryRecord(void);
 static void CheckPointGuts(XLogRecPtr checkPointRedo, int flags);
+static XLogSegNo GetOldestKeepSegment(XLogRecPtr currpos, XLogRecPtr minSlotPtr);
 static void KeepLogSeg(XLogRecPtr recptr, XLogSegNo *logSegNo);
 static XLogRecPtr XLogGetReplicationSlotMinimumLSN(void);
 
@@ -3928,6 +3930,55 @@ XLogGetLastRemovedSegno(void)
 	return lastRemovedSegNo;
 }
 
+/*
+ * Return the oldest WAL segment file.
+ *
+ * The returned value is XLogGetLastRemovedSegno() + 1 when that function
+ * returns a valid value.  Otherwise this function scans over WAL files and
+ * finds the oldest segment at the first time, which could be very slow.
+ */
+XLogSegNo
+FindOldestXLogFileSegNo(void)
+{
+	static XLogSegNo lastFoundOldestSeg = 0;
+	DIR		*xldir;
+	struct dirent *xlde;
+	XLogSegNo segno = XLogGetLastRemovedSegno();
+
+	if (segno > 0)
+		return segno + 1;
+
+	if (lastFoundOldestSeg > 0)
+		return lastFoundOldestSeg;
+
+	xldir = AllocateDir(XLOGDIR);
+	while ((xlde = ReadDir(xldir, XLOGDIR)) != NULL)
+	{
+		TimeLineID tli;
+		XLogSegNo fsegno;
+
+		/* Ignore files that are not XLOG segments */
+		if (!IsXLogFileName(xlde->d_name) &&
+			!IsPartialXLogFileName(xlde->d_name))
+			continue;
+
+		XLogFromFileName(xlde->d_name, &tli, &fsegno, wal_segment_size);
+
+		/*
+		 * Get minimum segment ignoring timeline ID, the same way with
+		 * RemoveOldXlogFiles().
+		 */
+		if (segno == 0 || fsegno < segno)
+			segno = fsegno;
+	}
+
+	FreeDir(xldir);
+
+	lastFoundOldestSeg = segno;
+
+	return segno;
+}
+
 /*
  * Update the last removed segno pointer in shared memory, to reflect
  * that the given XLOG file has been removed.
@@ -9441,6 +9492,197 @@ CreateRestartPoint(int flags)
 	return true;
 }
 
+/*
+ * Detect availability of the record at given targetLSN.
+ *
+ * targetLSN is restart_lsn of a slot.
+ * walsender_pid is the slot's walsender PID.
+ *
+ * Returns one of the following enum values.
+ *
+ * WALAVAIL_NORMAL means targetLSN is available because it is in the range of
+ * max_wal_size.  If max_slot_wal_keep_size is smaller than max_wal_size, this
+ * state is not returned.
+ *
+ * WALAVAIL_PRESERVED means it is still available by preserving extra segments
+ * beyond max_wal_size.
+ *
+ * WALAVAIL_BEING_REMOVED means it is being removed or already removed but the
+ * replication stream on the given slot is live yet. The state may transit to
+ * WALAVAIL_PRESERVED or WALAVAIL_NORMAL state if the walsender advances
+ * restart_lsn.
+ *
+ * WALAVAIL_REMOVED means it is definitly lost. The replication stream on the
+ * slot cannot continue.
+ *
+ * returns WALAVAIL_NULL if restart_lsn is invalid.
+ */
+WalAvailability
+GetWalAvailability(XLogRecPtr restart_lsn, pid_t walsender_pid)
+{
+	XLogRecPtr currpos;
+	XLogRecPtr slotPtr;
+	XLogSegNo currSeg;		/* segid of currpos */
+	XLogSegNo restartSeg;	/* segid of restart_lsn */
+	XLogSegNo oldestSeg;	/* actual oldest segid */
+	XLogSegNo oldestSegMaxWalSize;	/* oldest segid kept by max_wal_size */
+	XLogSegNo oldestSlotSeg;/* oldest segid kept by slot */
+	uint64	  keepSegs;
+
+	/* the case where the slot has never been activated */
+	if (XLogRecPtrIsInvalid(restart_lsn))
+		return WALAVAIL_INVALID_LSN;
+
+	currpos = GetXLogWriteRecPtr();
+
+	/* calculate oldest segment currently needed by slots */
+	XLByteToSeg(restart_lsn, restartSeg, wal_segment_size);
+	slotPtr = XLogGetReplicationSlotMinimumLSN();
+	oldestSlotSeg = GetOldestKeepSegment(currpos, slotPtr);
+
+	/* find the oldest extant segment file */
+	oldestSeg = FindOldestXLogFileSegNo();
+
+	/* calculate oldest segment by max_wal_size */
+	XLByteToSeg(currpos, currSeg, wal_segment_size);
+	keepSegs = ConvertToXSegs(max_wal_size_mb, wal_segment_size) + 1;
+
+	if (currSeg > keepSegs)
+		oldestSegMaxWalSize = currSeg - keepSegs;
+	else
+		oldestSegMaxWalSize = 1;
+
+	/*
+	 * If max_slot_wal_keep_size has changed after the last call, the segment
+	 * that would been kept by the current setting might have been lost by the
+	 * previous setting. No point in showing normal or keeping status values if
+	 * the restartSeg is known to be lost.
+	 */
+	if (restartSeg >= oldestSeg)
+	{
+		/*
+		 * show "normal" when restartSeg is within max_wal_size. If
+		 * max_slot_wal_keep_size is smaller than max_wal_size, there's no
+		 * point in showing the status.
+		 */
+		if ((max_slot_wal_keep_size_mb <= 0 ||
+			 max_slot_wal_keep_size_mb >= max_wal_size_mb) &&
+			oldestSegMaxWalSize <= restartSeg)
+			return WALAVAIL_NORMAL;
+
+		/* being retained by slots */
+		if (oldestSlotSeg <= restartSeg)
+			return WALAVAIL_PRESERVED;
+	}
+	
+	/*
+	 * The segment is already lost or being lost. If the oldest segment is just
+	 * after the restartSeg, running walsender may be reading the just removed
+	 * segment. The walsender may safely move to the oldest existing segment in
+	 * that case.
+	 */
+	if (oldestSeg == restartSeg + 1 && walsender_pid != 0)
+		return WALAVAIL_BEING_REMOVED;
+
+	/* definitely lost. the walsender can no longer restart */
+	return WALAVAIL_REMOVED;
+}
+
+/*
+ * Returns minimum segment number that the next checkpoint must leave
+ * considering wal_keep_segments, replication slots and
+ * max_slot_wal_keep_size.
+ *
+ * currLSN is the current insert location.
+ * minSlotLSN is the minimum restart_lsn of all active slots.
+ */
+static XLogSegNo
+GetOldestKeepSegment(XLogRecPtr currLSN, XLogRecPtr minSlotLSN)
+{
+	XLogSegNo	currSeg;
+	XLogSegNo	minSlotSeg;
+	uint64		keepSegs = 0;	/* # of segments actually kept */
+
+	XLByteToSeg(currLSN, currSeg, wal_segment_size);
+	XLByteToSeg(minSlotLSN, minSlotSeg, wal_segment_size);
+
+	/*
+	 * Calculate how many segments are kept by slots first. The second
+	 * term of the condition is just a sanity check.
+	 */
+	if (minSlotLSN != InvalidXLogRecPtr && minSlotSeg <= currSeg)
+		keepSegs = currSeg - minSlotSeg;
+
+	/* Cap keepSegs by max_slot_wal_keep_size */
+	if (max_slot_wal_keep_size_mb >= 0)
+	{
+		uint64 limitSegs;
+
+		limitSegs = ConvertToXSegs(max_slot_wal_keep_size_mb, wal_segment_size);
+
+		/* Reduce it if slots already reserves too many. */
+		if (limitSegs < keepSegs)
+			keepSegs = limitSegs;
+	}
+
+	/* but, keep at least wal_keep_segments segments if any */
+	if (wal_keep_segments > 0 && keepSegs < wal_keep_segments)
+		keepSegs = wal_keep_segments;
+
+	/* avoid underflow, don't go below 1 */
+	if (currSeg <= keepSegs)
+		return 1;
+
+	return currSeg - keepSegs;
+}
+
+/*
+ * Calculate remaining bytes until WAL segment for targetLSN will be removed.
+ */
+int64
+DistanceToWalRemoval(XLogRecPtr currLSN, XLogRecPtr targetLSN)
+{
+	XLogSegNo	currSeg;
+	uint64		limitSegs = 0;
+	int64 		restbytes;
+	uint64		fragbytes;
+	XLogSegNo	targetSeg;
+
+	XLByteToSeg(currLSN, currSeg, wal_segment_size);
+
+	/* Calculate how far back WAL segments are preserved */
+	if (max_slot_wal_keep_size_mb >= 0)
+		limitSegs = ConvertToXSegs(max_slot_wal_keep_size_mb, wal_segment_size);
+
+	if (wal_keep_segments > 0 && limitSegs < wal_keep_segments)
+		limitSegs = wal_keep_segments;
+
+	XLByteToSeg(targetLSN, targetSeg, wal_segment_size);
+
+	/* avoid underflow */
+	if (targetSeg + limitSegs < currSeg)
+		return 0;
+
+	/*
+	 * This slot still has all required segments. Calculate how
+	 * many LSN bytes the slot has until it loses targetLSN.
+	 */
+	fragbytes = wal_segment_size - (currLSN % wal_segment_size);
+	XLogSegNoOffsetToRecPtr(targetSeg + limitSegs - currSeg,
+							fragbytes, wal_segment_size,
+							restbytes);
+
+	/*
+	 * not realistic, but make sure that it is not out of the
+	 * range of int64. No problem to do so since such large values
+	 * have no significant difference.
+	 */
+	if (restbytes > PG_INT64_MAX)
+		restbytes = PG_INT64_MAX;
+
+	return restbytes;
+}
+
 /*
  * Retreat *logSegNo to the last segment that we need to retain because of
  * either wal_keep_segments or replication slots.
@@ -9452,38 +9694,102 @@ CreateRestartPoint(int flags)
 static void
 KeepLogSeg(XLogRecPtr recptr, XLogSegNo *logSegNo)
 {
-	XLogSegNo	segno;
-	XLogRecPtr	keep;
+	static XLogSegNo last_lost_segs = 0;
+	static int last_nslots = 0;
+	static char *last_slot_name = NULL;
+	XLogRecPtr	slotminptr = InvalidXLogRecPtr;
+	XLogSegNo	minSegNo;
+	XLogSegNo	minSlotSegNo;
+	int			nslots_affected = 0;
 
-	XLByteToSeg(recptr, segno, wal_segment_size);
-	keep = XLogGetReplicationSlotMinimumLSN();
+	if (max_replication_slots > 0)
+		slotminptr = XLogGetReplicationSlotMinimumLSN();
 
-	/* compute limit for wal_keep_segments first */
-	if (wal_keep_segments > 0)
+	/*
+	 * We should keep certain number of WAL segments after this checkpoint.
+	 */
+	minSegNo = GetOldestKeepSegment(recptr, slotminptr);
+
+	/*
+	 * Warn the checkpoint is going to flush the segments required by
+	 * replication slots.
+	 */
+	if (!XLogRecPtrIsInvalid(slotminptr))
 	{
-		/* avoid underflow, don't go below 1 */
-		if (segno <= wal_keep_segments)
-			segno = 1;
-		else
-			segno = segno - wal_keep_segments;
+		Assert (max_replication_slots > 0);
+
+		XLByteToSeg(slotminptr, minSlotSegNo, wal_segment_size);
+
+		if (minSlotSegNo < minSegNo)
+		{
+			/* Some slots has lost requred segments */
+			XLogSegNo	lost_segs = minSegNo - minSlotSegNo;
+			ReplicationSlot *earliest = NULL;
+			char	   *earliest_name = NULL;
+			int			i;
+
+			/* Find the most affected slot */
+			LWLockAcquire(ReplicationSlotControlLock, LW_SHARED);
+			for (i = 0 ; i < max_replication_slots ; i++)
+			{
+				ReplicationSlot *s =
+					&ReplicationSlotCtl->replication_slots[i];
+				XLogSegNo slotSegNo;
+
+				XLByteToSeg(s->data.restart_lsn, slotSegNo, wal_segment_size);
+
+				if (s->in_use && s->active_pid == 0 && slotSegNo < minSegNo)
+				{
+					nslots_affected++;
+
+					if (earliest == NULL ||
+						s->data.restart_lsn < earliest->data.restart_lsn)
+						earliest = s;
+				}
+			}
+
+			if (earliest)
+			{
+				MemoryContext oldcxt = MemoryContextSwitchTo(TopMemoryContext);
+				earliest_name = pstrdup(NameStr(earliest->data.name));
+				MemoryContextSwitchTo(oldcxt);
+			}
+
+			LWLockRelease(ReplicationSlotControlLock);
+
+			/* Emit WARNING if something has changed */
+			if (earliest_name &&
+				(last_lost_segs != lost_segs || last_nslots != nslots_affected))
+			{
+				ereport(WARNING,
+						(errmsg_plural ("%d replication slot has lost required WAL segments by %lu segments",
+										"%d replication slots have lost required WAL segments by %lu segments",
+										nslots_affected, nslots_affected,
+										lost_segs),
+						 errdetail("Most affected slot is %s.",
+								   earliest_name)));
+
+				if (last_slot_name)
+					pfree(last_slot_name);
+				last_slot_name = earliest_name;
+				last_lost_segs = lost_segs;
+				last_nslots = nslots_affected;
+			}
+		}
 	}
 
-	/* then check whether slots limit removal further */
-	if (max_replication_slots > 0 && keep != InvalidXLogRecPtr)
+	/* Reset the state if no affected slots remain. */
+	if (nslots_affected == 0 && last_slot_name)
 	{
-		XLogSegNo	slotSegNo;
-
-		XLByteToSeg(keep, slotSegNo, wal_segment_size);
-
-		if (slotSegNo <= 0)
-			segno = 1;
-		else if (slotSegNo < segno)
-			segno = slotSegNo;
+		pfree(last_slot_name);
+		last_slot_name = NULL;
+		last_lost_segs = 0;
+		last_nslots = 0;
 	}
 
 	/* don't delete WAL segments newer than the calculated segment */
-	if (segno < *logSegNo)
-		*logSegNo = segno;
+	if (minSegNo < *logSegNo)
+		*logSegNo = minSegNo;
 }
 
 /*
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index 83d00c6cde..775b8b7f20 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -863,7 +863,9 @@ CREATE VIEW pg_replication_slots AS
             L.xmin,
             L.catalog_xmin,
             L.restart_lsn,
-            L.confirmed_flush_lsn
+            L.confirmed_flush_lsn,
+            L.wal_status,
+            L.remain
     FROM pg_get_replication_slots() AS L
             LEFT JOIN pg_database D ON (L.datoid = D.oid);
 
diff --git a/src/backend/replication/slot.c b/src/backend/replication/slot.c
index d90c7235e9..a26f7999aa 100644
--- a/src/backend/replication/slot.c
+++ b/src/backend/replication/slot.c
@@ -49,6 +49,7 @@
 #include "storage/proc.h"
 #include "storage/procarray.h"
 #include "utils/builtins.h"
+#include "utils/memutils.h"
 
 /*
  * Replication slot on-disk data structure.
diff --git a/src/backend/replication/slotfuncs.c b/src/backend/replication/slotfuncs.c
index ce0c9127bc..47cd4375a1 100644
--- a/src/backend/replication/slotfuncs.c
+++ b/src/backend/replication/slotfuncs.c
@@ -234,7 +234,7 @@ pg_drop_replication_slot(PG_FUNCTION_ARGS)
 Datum
 pg_get_replication_slots(PG_FUNCTION_ARGS)
 {
-#define PG_GET_REPLICATION_SLOTS_COLS 11
+#define PG_GET_REPLICATION_SLOTS_COLS 13
 	ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo;
 	TupleDesc	tupdesc;
 	Tuplestorestate *tupstore;
@@ -288,6 +288,7 @@ pg_get_replication_slots(PG_FUNCTION_ARGS)
 		Oid			database;
 		NameData	slot_name;
 		NameData	plugin;
+		WalAvailability walstate;
 		int			i;
 
 		if (!slot->in_use)
@@ -355,6 +356,42 @@ pg_get_replication_slots(PG_FUNCTION_ARGS)
 		else
 			nulls[i++] = true;
 
+		walstate = GetWalAvailability(restart_lsn, active_pid);
+
+		switch (walstate)
+		{
+			case WALAVAIL_INVALID_LSN:
+				nulls[i++] = true;
+				break;
+
+			case WALAVAIL_NORMAL:
+				values[i++] = CStringGetTextDatum("normal");
+				break;
+
+			case WALAVAIL_PRESERVED:
+				values[i++] = CStringGetTextDatum("keeping");
+				break;
+
+			case WALAVAIL_BEING_REMOVED:
+				values[i++] = CStringGetTextDatum("losing");
+				break;
+
+			case WALAVAIL_REMOVED:
+				values[i++] = CStringGetTextDatum("lost");
+				break;
+		}
+
+		if (max_slot_wal_keep_size_mb >= 0 &&
+			(walstate == WALAVAIL_NORMAL ||
+			 walstate == WALAVAIL_PRESERVED))
+		{
+			values[i++] =
+				Int64GetDatum(DistanceToWalRemoval(GetXLogWriteRecPtr(),
+												   restart_lsn));
+		}
+		else
+			nulls[i++] = true;
+
 		tuplestore_putvalues(tupstore, tupdesc, values, nulls);
 	}
 	LWLockRelease(ReplicationSlotControlLock);
diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c
index 79bc7ac8ca..54cd5f6420 100644
--- a/src/backend/utils/misc/guc.c
+++ b/src/backend/utils/misc/guc.c
@@ -2771,6 +2771,19 @@ static struct config_int ConfigureNamesInt[] =
 		NULL, NULL, NULL
 	},
 
+	{
+		{"max_slot_wal_keep_size", PGC_SIGHUP, REPLICATION_SENDING,
+			gettext_noop("Sets the maximum size of WAL space reserved by replication slots."),
+			gettext_noop("Replication slots will be marked as failed, and segments released "
+						 "for deletion or recycling, if this much space is occupied by WAL "
+						 "on disk."),
+			GUC_UNIT_MB
+		},
+		&max_slot_wal_keep_size_mb,
+		-1, -1, MAX_KILOBYTES,
+		NULL, NULL, NULL
+	},
+
 	{
 		{"wal_sender_timeout", PGC_USERSET, REPLICATION_SENDING,
 			gettext_noop("Sets the maximum time to wait for WAL replication."),
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index e9f8ca775d..0b696e7044 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -287,6 +287,7 @@
 #max_wal_senders = 10		# max number of walsender processes
 				# (change requires restart)
 #wal_keep_segments = 0		# in logfile segments; 0 disables
+#max_slot_wal_keep_size = -1	# measured in bytes; -1 disables
 #wal_sender_timeout = 60s	# in milliseconds; 0 disables
 
 #max_replication_slots = 10	# max number of replication slots
diff --git a/src/include/access/xlog.h b/src/include/access/xlog.h
index 9ec7b31cce..16d33b41f7 100644
--- a/src/include/access/xlog.h
+++ b/src/include/access/xlog.h
@@ -108,6 +108,7 @@ extern int	wal_segment_size;
 extern int	min_wal_size_mb;
 extern int	max_wal_size_mb;
 extern int	wal_keep_segments;
+extern int	max_slot_wal_keep_size_mb;
 extern int	XLOGbuffers;
 extern int	XLogArchiveTimeout;
 extern int	wal_retrieve_retry_interval;
@@ -255,6 +256,20 @@ typedef struct CheckpointStatsData
 
 extern CheckpointStatsData CheckpointStats;
 
+/*
+ * WAL segment availability status
+ *
+ * This is used as the return value of GetWalAvailability.
+ */
+typedef enum WalAvailability
+{
+	WALAVAIL_INVALID_LSN,			/* parameter errror */
+	WALAVAIL_NORMAL,				/* WAL segment is within max_wal_size */
+	WALAVAIL_PRESERVED,				/* WAL segment is preserved by repslots */
+	WALAVAIL_BEING_REMOVED,			/* WAL segment is no longer preserved */
+	WALAVAIL_REMOVED				/* WAL segment has been removed */
+} WalAvailability;
+
 struct XLogRecData;
 
 extern XLogRecPtr XLogInsertRecord(struct XLogRecData *rdata,
@@ -268,6 +283,7 @@ extern int	XLogFileOpen(XLogSegNo segno);
 
 extern void CheckXLogRemoved(XLogSegNo segno, TimeLineID tli);
 extern XLogSegNo XLogGetLastRemovedSegno(void);
+extern XLogSegNo FindOldestXLogFileSegNo(void);
 extern void XLogSetAsyncXactLSN(XLogRecPtr record);
 extern void XLogSetReplicationSlotMinimumLSN(XLogRecPtr lsn);
 
@@ -305,6 +321,9 @@ extern void ShutdownXLOG(int code, Datum arg);
 extern void InitXLOGAccess(void);
 extern void CreateCheckPoint(int flags);
 extern bool CreateRestartPoint(int flags);
+extern WalAvailability GetWalAvailability(XLogRecPtr restart_lsn,
+										  pid_t walsender_pid);
+extern int64 DistanceToWalRemoval(XLogRecPtr currLSN, XLogRecPtr targetLSN);
 extern void XLogPutNextOid(Oid nextOid);
 extern XLogRecPtr XLogRestorePoint(const char *rpName);
 extern void UpdateFullPageWrites(void);
diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat
index a6a708cca9..2025f34bfd 100644
--- a/src/include/catalog/pg_proc.dat
+++ b/src/include/catalog/pg_proc.dat
@@ -9971,9 +9971,9 @@
   proname => 'pg_get_replication_slots', prorows => '10', proisstrict => 'f',
   proretset => 't', provolatile => 's', prorettype => 'record',
   proargtypes => '',
-  proallargtypes => '{name,name,text,oid,bool,bool,int4,xid,xid,pg_lsn,pg_lsn}',
-  proargmodes => '{o,o,o,o,o,o,o,o,o,o,o}',
-  proargnames => '{slot_name,plugin,slot_type,datoid,temporary,active,active_pid,xmin,catalog_xmin,restart_lsn,confirmed_flush_lsn}',
+  proallargtypes => '{name,name,text,oid,bool,bool,int4,xid,xid,pg_lsn,pg_lsn,text,int8}',
+  proargmodes => '{o,o,o,o,o,o,o,o,o,o,o,o,o}',
+  proargnames => '{slot_name,plugin,slot_type,datoid,temporary,active,active_pid,xmin,catalog_xmin,restart_lsn,confirmed_flush_lsn,wal_status,remain}',
   prosrc => 'pg_get_replication_slots' },
 { oid => '3786', descr => 'set up a logical replication slot',
   proname => 'pg_create_logical_replication_slot', provolatile => 'v',
diff --git a/src/test/recovery/t/018_replslot_limit.pl b/src/test/recovery/t/018_replslot_limit.pl
new file mode 100644
index 0000000000..6688167546
--- /dev/null
+++ b/src/test/recovery/t/018_replslot_limit.pl
@@ -0,0 +1,202 @@
+# Test for replication slot limit
+# Ensure that max_slot_wal_keep_size limits the number of WAL files to
+# be kept by replication slots.
+
+use strict;
+use warnings;
+use File::Path qw(rmtree);
+use PostgresNode;
+use TestLib;
+use Test::More tests => 13;
+use Time::HiRes qw(usleep);
+
+$ENV{PGDATABASE} = 'postgres';
+
+# Initialize master node, setting wal-segsize to 1MB
+my $node_master = get_new_node('master');
+$node_master->init(allows_streaming => 1, extra => ['--wal-segsize=1']);
+$node_master->append_conf('postgresql.conf', qq(
+min_wal_size = 2MB
+max_wal_size = 4MB
+log_checkpoints = yes
+));
+$node_master->start;
+$node_master->safe_psql('postgres', "SELECT pg_create_physical_replication_slot('rep1')");
+
+# The slot state and remain should be null before the first connection
+my $result = $node_master->safe_psql('postgres', "SELECT restart_lsn is NULL, wal_status is NULL, remain is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'");
+is($result, "t|t|t", 'check the state of non-reserved slot is "unknown"');
+
+
+# Take backup
+my $backup_name = 'my_backup';
+$node_master->backup($backup_name);
+
+# Create a standby linking to it using the replication slot
+my $node_standby = get_new_node('standby_1');
+$node_standby->init_from_backup($node_master, $backup_name, has_streaming => 1, primary_slot_name => 'rep1');
+
+$node_standby->start;
+
+# Wait until standby has replayed enough data
+my $start_lsn = $node_master->lsn('write');
+$node_master->wait_for_catchup($node_standby, 'replay', $start_lsn);
+
+# Stop standby
+$node_standby->stop;
+
+
+# Preparation done, the slot is the state "normal" now
+$result = $node_master->safe_psql('postgres', "SELECT restart_lsn, wal_status, remain is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'");
+is($result, "$start_lsn|normal|t", 'check the catching-up state');
+
+# Advance WAL by five segments (= 5MB) on master
+advance_wal($node_master, 1);
+$node_master->safe_psql('postgres', "CHECKPOINT;");
+
+# The slot is always "safe" when fitting max_wal_size
+$result = $node_master->safe_psql('postgres', "SELECT restart_lsn, wal_status, remain is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'");
+is($result, "$start_lsn|normal|t", 'check that restart_lsn is in max_wal_size');
+
+advance_wal($node_master, 4);
+$node_master->safe_psql('postgres', "CHECKPOINT;");
+
+# The slot is always "safe" when max_slot_wal_keep_size is not set
+$result = $node_master->safe_psql('postgres', "SELECT restart_lsn, wal_status, remain is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'");
+is($result, "$start_lsn|normal|t", 'check that slot is working');
+
+# The standby can reconnect to master
+$node_standby->start;
+
+$start_lsn = $node_master->lsn('write');
+$node_master->wait_for_catchup($node_standby, 'replay', $start_lsn);
+
+$node_standby->stop;
+
+# Set max_slot_wal_keep_size on master
+my $max_slot_wal_keep_size_mb = 6;
+$node_master->append_conf('postgresql.conf', qq(
+max_slot_wal_keep_size = ${max_slot_wal_keep_size_mb}MB
+));
+$node_master->reload;
+
+# The slot is in safe state. The remaining bytes should be as almost
+# (max_slot_wal_keep_size + 1) times large as the segment size
+$result = $node_master->safe_psql('postgres', "SELECT restart_lsn, wal_status, pg_size_pretty(remain) as remain FROM pg_replication_slots WHERE slot_name = 'rep1'");
+is($result, "$start_lsn|normal|7168 kB", 'check that max_slot_wal_keep_size is working');
+
+# Advance WAL again then checkpoint, reducing remain by 2 MB.
+advance_wal($node_master, 2);
+$node_master->safe_psql('postgres', "CHECKPOINT;");
+
+# The slot is still working
+$result = $node_master->safe_psql('postgres', "SELECT restart_lsn, wal_status, pg_size_pretty(remain) as remain FROM pg_replication_slots WHERE slot_name = 'rep1'");
+is($result, "$start_lsn|normal|5120 kB", 'check that remaining byte is calculated correctly');
+
+# wal_keep_segments overrides max_slot_wal_keep_size
+$result = $node_master->safe_psql('postgres', "ALTER SYSTEM SET wal_keep_segments to 8; SELECT pg_reload_conf();");
+$result = $node_master->safe_psql('postgres', "SELECT restart_lsn, wal_status, pg_size_pretty(remain) as remain FROM pg_replication_slots WHERE slot_name = 'rep1'");
+is($result, "$start_lsn|normal|7168 kB", 'check that wal_keep_segments overrides max_slot_wal_keep_size');
+
+# restore wal_keep_segments
+$result = $node_master->safe_psql('postgres', "ALTER SYSTEM SET wal_keep_segments to 0; SELECT pg_reload_conf();");
+
+# Advance WAL again without checkpoint, reducing remain by 4 MB.
+advance_wal($node_master, 4);
+
+# Slot gets into 'keeping' state
+$result = $node_master->safe_psql('postgres', "SELECT restart_lsn, wal_status, pg_size_pretty(remain) as remain FROM pg_replication_slots WHERE slot_name = 'rep1'");
+is($result, "$start_lsn|keeping|1024 kB", 'check that the slot state changes to "keeping"');
+
+# do checkpoint so that the next checkpoint runs too early
+$node_master->safe_psql('postgres', "CHECKPOINT;");
+
+# Advance WAL again without checkpoint; remain goes to 0.
+advance_wal($node_master, 1);
+
+# Slot gets into 'lost' state
+$result = $node_master->safe_psql('postgres', "SELECT restart_lsn, wal_status, remain is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'");
+is($result, "$start_lsn|lost|t", 'check that the slot state changes to "lost"');
+
+# The standby still can connect to master before a checkpoint
+$node_standby->start;
+
+$start_lsn = $node_master->lsn('write');
+$node_master->wait_for_catchup($node_standby, 'replay', $start_lsn);
+
+$node_standby->stop;
+
+ok(!find_in_log($node_standby,
+				"requested WAL segment [0-9A-F]+ has already been removed"),
+   'check that required WAL segments are still available');
+
+# Advance WAL again, the slot loses the oldest segment.
+my $logstart = get_log_size($node_master);
+advance_wal($node_master, 7);
+$node_master->safe_psql('postgres', "CHECKPOINT;");
+
+# WARNING should be issued
+ok(find_in_log($node_master,
+			   "1 replication slot has lost required WAL segments by 1 segments\n".
+			   ".*Most affected slot is rep1.",
+			   $logstart),
+   'check that the warning is logged');
+
+# This slot should be broken
+$result = $node_master->safe_psql('postgres', "SELECT restart_lsn, wal_status, remain is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'");
+is($result, "$start_lsn|lost|t", 'check that the slot state changes to "lost"');
+
+# The standby no longer can connect to the master
+$logstart = get_log_size($node_standby);
+$node_standby->start;
+
+my $failed = 0;
+for (my $i = 0 ; $i < 10000 ; $i++)
+{
+	if (find_in_log($node_standby,
+					"requested WAL segment [0-9A-F]+ has already been removed",
+					$logstart))
+	{
+		$failed = 1;
+		last;
+	}
+	usleep(100_000);
+}
+ok($failed, 'check that replication has been broken');
+
+$node_standby->stop;
+
+#####################################
+# Advance WAL of $node by $n segments
+sub advance_wal
+{
+	my ($node, $n) = @_;
+
+	# Advance by $n segments (= (16 * $n) MB) on master
+	for (my $i = 0 ; $i < $n ; $i++)
+	{
+		$node->safe_psql('postgres', "CREATE TABLE t (); DROP TABLE t; SELECT pg_switch_wal();");
+	}
+}
+
+# return the size of logfile of $node in bytes
+sub get_log_size
+{
+	my ($node) = @_;
+
+	return (stat $node->logfile)[7];
+}
+
+# find $pat in logfile of $node after $off-th byte
+sub find_in_log
+{
+	my ($node, $pat, $off) = @_;
+
+	$off = 0 unless defined $off;
+	my $log = TestLib::slurp_file($node->logfile);
+	return 0 if (length($log) <= $off);
+
+	$log = substr($log, $off);
+
+	return $log =~ m/$pat/;
+}
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index 7245b0e13b..8688f7138f 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1462,8 +1462,10 @@ pg_replication_slots| SELECT l.slot_name,
     l.xmin,
     l.catalog_xmin,
     l.restart_lsn,
-    l.confirmed_flush_lsn
-   FROM (pg_get_replication_slots() l(slot_name, plugin, slot_type, datoid, temporary, active, active_pid, xmin, catalog_xmin, restart_lsn, confirmed_flush_lsn)
+    l.confirmed_flush_lsn,
+    l.wal_status,
+    l.remain
+   FROM (pg_get_replication_slots() l(slot_name, plugin, slot_type, datoid, temporary, active, active_pid, xmin, catalog_xmin, restart_lsn, confirmed_flush_lsn, wal_status, remain)
      LEFT JOIN pg_database d ON ((l.datoid = d.oid)));
 pg_roles| SELECT pg_authid.rolname,
     pg_authid.rolsuper,
-- 
2.20.1


--d6Gm4EdcadzBjdND
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v19-0003-Check-removal-of-in-reading-segment-file.patch"



^ permalink  raw  reply  [nested|flat] 125+ messages in thread

* [PATCH v20 2/3] Add WAL relief vent for replication slots
@ 2017-12-21 12:20 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 125+ messages in thread

From: Kyotaro Horiguchi @ 2017-12-21 12:20 UTC (permalink / raw)

Replication slot is useful to maintain replication connection in the
configurations where replication is so delayed that connection is
broken. On the other hand so many WAL files can fill up disk that the
master downs by a long delay. This feature, which is activated by a
GUC "max_slot_wal_keep_size", protects master servers from suffering
disk full by limiting the number of WAL files reserved by replication
slots.
---
 contrib/test_decoding/expected/ddl.out        |   4 +-
 contrib/test_decoding/sql/ddl.sql             |   2 +
 doc/src/sgml/catalogs.sgml                    |  48 +++
 doc/src/sgml/config.sgml                      |  23 ++
 doc/src/sgml/high-availability.sgml           |   8 +-
 src/backend/access/transam/xlog.c             | 361 ++++++++++++++++--
 src/backend/catalog/system_views.sql          |   4 +-
 src/backend/replication/slot.c                |   1 +
 src/backend/replication/slotfuncs.c           |  39 +-
 src/backend/utils/misc/guc.c                  |  13 +
 src/backend/utils/misc/postgresql.conf.sample |   1 +
 src/include/access/xlog.h                     |  19 +
 src/include/catalog/pg_proc.dat               |   6 +-
 src/test/recovery/t/018_replslot_limit.pl     | 202 ++++++++++
 src/test/regress/expected/rules.out           |   6 +-
 15 files changed, 699 insertions(+), 38 deletions(-)
 create mode 100644 src/test/recovery/t/018_replslot_limit.pl

diff --git a/contrib/test_decoding/expected/ddl.out b/contrib/test_decoding/expected/ddl.out
index 2c999fd3eb..cf0318f697 100644
--- a/contrib/test_decoding/expected/ddl.out
+++ b/contrib/test_decoding/expected/ddl.out
@@ -723,8 +723,8 @@ SELECT pg_drop_replication_slot('regression_slot');
 (1 row)
 
 /* check that the slot is gone */
+\x
 SELECT * FROM pg_replication_slots;
- slot_name | plugin | slot_type | datoid | database | temporary | active | active_pid | xmin | catalog_xmin | restart_lsn | confirmed_flush_lsn 
------------+--------+-----------+--------+----------+-----------+--------+------------+------+--------------+-------------+---------------------
 (0 rows)
 
+\x
diff --git a/contrib/test_decoding/sql/ddl.sql b/contrib/test_decoding/sql/ddl.sql
index 856495c952..0f2b9992f7 100644
--- a/contrib/test_decoding/sql/ddl.sql
+++ b/contrib/test_decoding/sql/ddl.sql
@@ -387,4 +387,6 @@ SELECT data FROM pg_logical_slot_get_changes('regression_slot', NULL, NULL, 'inc
 SELECT pg_drop_replication_slot('regression_slot');
 
 /* check that the slot is gone */
+\x
 SELECT * FROM pg_replication_slots;
+\x
diff --git a/doc/src/sgml/catalogs.sgml b/doc/src/sgml/catalogs.sgml
index 64614b569c..01a7802ed4 100644
--- a/doc/src/sgml/catalogs.sgml
+++ b/doc/src/sgml/catalogs.sgml
@@ -9907,6 +9907,54 @@ SELECT * FROM pg_locks pl LEFT JOIN pg_prepared_xacts ppx
       </entry>
      </row>
 
+     <row>
+      <entry><structfield>wal_status</structfield></entry>
+      <entry><type>text</type></entry>
+      <entry></entry>
+
+      <entry>Availability of WAL files claimed by this slot.
+      Valid values are:
+       <simplelist>
+        <member>
+         <literal>normal</literal> means that the claimed files
+         are within <varname>max_wal_size</varname>
+        </member>
+        <member>
+         <literal>keeping</literal> means that <varname>max_wal_size</varname>
+         is exceeded but still held by replication slots or
+         <varname>wal_keep_segments</varname>
+        </member>
+        <member>
+         <literal>losing</literal> means that some of the files are on the verge
+         of deletion, but can still be accessed by a session that's currently
+         reading it
+        </member>
+        <member>
+         <literal>lost</literal> means that some of them are definitely lost
+         and the session using this slot cannot continue replication.
+         This state also implies that the session using this slot has been
+         stopped.
+        </member>
+       </simplelist>
+      The last two states are seen only when
+      <xref linkend="guc-max-slot-wal-keep-size"/> is
+      non-negative. If <structfield>restart_lsn</structfield> is NULL, this
+      field is null.
+      </entry>
+     </row>
+
+     <row>
+      <entry><structfield>remain</structfield></entry>
+      <entry><type>bigint</type></entry>
+      <entry></entry>
+      <entry>The amount in bytes of WAL that can be written before this slot
+        loses required WAL files.
+        If <structfield>restart_lsn</structfield> is null or
+        <structfield>wal_status</structfield> is <literal>losing</literal>
+        or <literal>lost</literal>, this field is null.
+      </entry>
+     </row>
+
     </tbody>
    </tgroup>
   </table>
diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index 2de21903a1..dc99c6868a 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -3758,6 +3758,29 @@ restore_command = 'copy "C:\\server\\archivedir\\%f" "%p"'  # Windows
       </listitem>
      </varlistentry>
 
+      <varlistentry id="guc-max-slot-wal-keep-size" xreflabel="max_slot_wal_keep_size">
+       <term><varname>max_slot_wal_keep_size</varname> (<type>integer</type>)
+       <indexterm>
+        <primary><varname>max_slot_wal_keep_size</varname> configuration parameter</primary>
+       </indexterm>
+       </term>
+       <listitem>
+       <para>
+        Specify the maximum size of WAL files
+        that <link linkend="streaming-replication-slots">replication
+        slots</link> are allowed to retain in the <filename>pg_wal</filename>
+        directory at checkpoint time.
+        If <varname>max_slot_wal_keep_size</varname> is -1 (the default),
+        replication slots retain unlimited amount of WAL files.  If
+        restart_lsn of a replication slot gets behind more than that megabytes
+        from the current LSN, the standby using the slot may no longer be able
+        to continue replication due to removal of required WAL files. You
+        can see the WAL availability of replication slots
+        in <link linkend="view-pg-replication-slots">pg_replication_slots</link>.
+       </para>
+       </listitem>
+      </varlistentry>
+
      <varlistentry id="guc-wal-sender-timeout" xreflabel="wal_sender_timeout">
       <term><varname>wal_sender_timeout</varname> (<type>integer</type>)
       <indexterm>
diff --git a/doc/src/sgml/high-availability.sgml b/doc/src/sgml/high-availability.sgml
index b5d32bb720..624e5f94ad 100644
--- a/doc/src/sgml/high-availability.sgml
+++ b/doc/src/sgml/high-availability.sgml
@@ -925,9 +925,11 @@ primary_conninfo = 'host=192.168.1.50 port=5432 user=foo password=foopass'
     <xref linkend="guc-archive-command"/>.
     However, these methods often result in retaining more WAL segments than
     required, whereas replication slots retain only the number of segments
-    known to be needed.  An advantage of these methods is that they bound
-    the space requirement for <literal>pg_wal</literal>; there is currently no way
-    to do this using replication slots.
+    known to be needed.  On the other hand, replication slots can retain so
+    many WAL segments that they fill up the space allocated
+    for <literal>pg_wal</literal>;
+    <xref linkend="guc-max-slot-wal-keep-size"/> limits the size of WAL files
+    retained by replication slots.
    </para>
    <para>
     Similarly, <xref linkend="guc-hot-standby-feedback"/>
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 1951103b26..e68a89fded 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -105,6 +105,7 @@ int			wal_level = WAL_LEVEL_MINIMAL;
 int			CommitDelay = 0;	/* precommit delay in microseconds */
 int			CommitSiblings = 5; /* # concurrent xacts needed to sleep */
 int			wal_retrieve_retry_interval = 5000;
+int			max_slot_wal_keep_size_mb = -1;
 
 #ifdef WAL_DEBUG
 bool		XLOG_DEBUG = false;
@@ -758,7 +759,7 @@ static ControlFileData *ControlFile = NULL;
  */
 #define UsableBytesInPage (XLOG_BLCKSZ - SizeOfXLogShortPHD)
 
-/* Convert min_wal_size_mb and max_wal_size_mb to equivalent segment count */
+/* Convert values of GUCs measured in megabytes to equiv. segment count */
 #define ConvertToXSegs(x, segsize)	\
 	(x / ((segsize) / (1024 * 1024)))
 
@@ -895,6 +896,7 @@ static void checkTimeLineSwitch(XLogRecPtr lsn, TimeLineID newTLI,
 static void LocalSetXLogInsertAllowed(void);
 static void CreateEndOfRecoveryRecord(void);
 static void CheckPointGuts(XLogRecPtr checkPointRedo, int flags);
+static XLogSegNo GetOldestKeepSegment(XLogRecPtr currpos, XLogRecPtr minSlotPtr);
 static void KeepLogSeg(XLogRecPtr recptr, XLogSegNo *logSegNo);
 static XLogRecPtr XLogGetReplicationSlotMinimumLSN(void);
 
@@ -3929,8 +3931,56 @@ XLogGetLastRemovedSegno(void)
 }
 
 /*
- * Update the last removed segno pointer in shared memory, to reflect
- * that the given XLOG file has been removed.
+ * Scan the WAL directory then set the lastRemovedSegNo.
+ *
+ * In the case we need to know the last removed segment before the first
+ * checkpoint runs, call this function to initialize the variable by scanning
+ * the WAL directory.
+ */
+XLogSegNo
+ScanAndSetLastRemovedSegno(void)
+{
+	DIR		*xldir;
+	struct dirent *xlde;
+	XLogSegNo segno;
+
+	xldir = AllocateDir(XLOGDIR);
+	while ((xlde = ReadDir(xldir, XLOGDIR)) != NULL)
+	{
+		TimeLineID tli;
+		XLogSegNo fsegno;
+
+		/* Ignore files that are not XLOG segments */
+		if (!IsXLogFileName(xlde->d_name) &&
+			!IsPartialXLogFileName(xlde->d_name))
+			continue;
+
+		XLogFromFileName(xlde->d_name, &tli, &fsegno, wal_segment_size);
+
+		/*
+		 * Get minimum segment ignoring timeline ID, the same way with
+		 * RemoveOldXlogFiles().
+		 */
+		if (segno == 0 || fsegno < segno)
+			segno = fsegno;
+	}
+
+	FreeDir(xldir);
+
+	/* Update the last removed segno, not making retrogression. */
+	SpinLockAcquire(&XLogCtl->info_lck);
+	if (segno > XLogCtl->lastRemovedSegNo)
+		XLogCtl->lastRemovedSegNo = segno;
+	else
+		segno = XLogCtl->lastRemovedSegNo;
+	SpinLockRelease(&XLogCtl->info_lck);
+
+	return segno;
+}
+
+/*
+ * Update the last removed segno pointer in shared memory, to reflect that the
+ * given XLOG file has been removed.
  */
 static void
 UpdateLastRemovedPtr(char *filename)
@@ -9441,6 +9491,201 @@ CreateRestartPoint(int flags)
 	return true;
 }
 
+/*
+ * Detect availability of the record at given targetLSN.
+ *
+ * targetLSN is restart_lsn of a slot.
+ * walsender_pid is the slot's walsender PID.
+ *
+ * Returns one of the following enum values.
+ *
+ * WALAVAIL_NORMAL means targetLSN is available because it is in the range of
+ * max_wal_size.  If max_slot_wal_keep_size is smaller than max_wal_size, this
+ * state is not returned.
+ *
+ * WALAVAIL_PRESERVED means it is still available by preserving extra segments
+ * beyond max_wal_size.
+ *
+ * WALAVAIL_BEING_REMOVED means it is being removed or already removed but the
+ * replication stream on the given slot is live yet. The state may transit to
+ * WALAVAIL_PRESERVED or WALAVAIL_NORMAL state if the walsender advances
+ * restart_lsn.
+ *
+ * WALAVAIL_REMOVED means it is definitely lost. The replication stream on the
+ * slot cannot continue.
+ *
+ * returns WALAVAIL_NULL if restart_lsn is invalid.
+ */
+WalAvailability
+GetWalAvailability(XLogRecPtr restart_lsn, pid_t walsender_pid)
+{
+	XLogRecPtr currpos;
+	XLogRecPtr slotPtr;
+	XLogSegNo currSeg;		/* segid of currpos */
+	XLogSegNo restartSeg;	/* segid of restart_lsn */
+	XLogSegNo oldestSeg;	/* actual oldest segid */
+	XLogSegNo oldestSegMaxWalSize;	/* oldest segid kept by max_wal_size */
+	XLogSegNo oldestSlotSeg;/* oldest segid kept by slot */
+	uint64	  keepSegs;
+
+	/* the case where the slot has never been activated */
+	if (XLogRecPtrIsInvalid(restart_lsn))
+		return WALAVAIL_INVALID_LSN;
+
+	currpos = GetXLogWriteRecPtr();
+
+	/* calculate oldest segment currently needed by slots */
+	XLByteToSeg(restart_lsn, restartSeg, wal_segment_size);
+	slotPtr = XLogGetReplicationSlotMinimumLSN();
+	oldestSlotSeg = GetOldestKeepSegment(currpos, slotPtr);
+
+	/* find the oldest extant segment file */
+	oldestSeg = XLogGetLastRemovedSegno() + 1;
+
+	/* initialize last removed segno if not yet */
+	if (oldestSeg == 1)
+		oldestSeg = ScanAndSetLastRemovedSegno() + 1;
+
+	/* calculate oldest segment by max_wal_size */
+	XLByteToSeg(currpos, currSeg, wal_segment_size);
+	keepSegs = ConvertToXSegs(max_wal_size_mb, wal_segment_size) + 1;
+
+	if (currSeg > keepSegs)
+		oldestSegMaxWalSize = currSeg - keepSegs;
+	else
+		oldestSegMaxWalSize = 1;
+
+	/*
+	 * If max_slot_wal_keep_size has changed after the last call, the segment
+	 * that would been kept by the current setting might have been lost by the
+	 * previous setting. No point in showing normal or keeping status values if
+	 * the restartSeg is known to be lost.
+	 */
+	if (restartSeg >= oldestSeg)
+	{
+		/*
+		 * show "normal" when restartSeg is within max_wal_size. If
+		 * max_slot_wal_keep_size is smaller than max_wal_size, there's no
+		 * point in showing the status.
+		 */
+		if ((max_slot_wal_keep_size_mb <= 0 ||
+			 max_slot_wal_keep_size_mb >= max_wal_size_mb) &&
+			oldestSegMaxWalSize <= restartSeg)
+			return WALAVAIL_NORMAL;
+
+		/* being retained by slots */
+		if (oldestSlotSeg <= restartSeg)
+			return WALAVAIL_PRESERVED;
+	}
+	
+	/*
+	 * The segment is already lost or being lost. If the oldest segment is just
+	 * after the restartSeg, running walsender may be reading the just removed
+	 * segment. The walsender may safely move to the oldest existing segment in
+	 * that case.
+	 */
+	if (oldestSeg == restartSeg + 1 && walsender_pid != 0)
+		return WALAVAIL_BEING_REMOVED;
+
+	/* definitely lost. the walsender can no longer restart */
+	return WALAVAIL_REMOVED;
+}
+
+/*
+ * Returns minimum segment number that the next checkpoint must leave
+ * considering wal_keep_segments, replication slots and
+ * max_slot_wal_keep_size.
+ *
+ * currLSN is the current insert location.
+ * minSlotLSN is the minimum restart_lsn of all active slots.
+ */
+static XLogSegNo
+GetOldestKeepSegment(XLogRecPtr currLSN, XLogRecPtr minSlotLSN)
+{
+	XLogSegNo	currSeg;
+	XLogSegNo	minSlotSeg;
+	uint64		keepSegs = 0;	/* # of segments actually kept */
+
+	XLByteToSeg(currLSN, currSeg, wal_segment_size);
+	XLByteToSeg(minSlotLSN, minSlotSeg, wal_segment_size);
+
+	/*
+	 * Calculate how many segments are kept by slots first. The second
+	 * term of the condition is just a sanity check.
+	 */
+	if (minSlotLSN != InvalidXLogRecPtr && minSlotSeg <= currSeg)
+		keepSegs = currSeg - minSlotSeg;
+
+	/* Cap keepSegs by max_slot_wal_keep_size */
+	if (max_slot_wal_keep_size_mb >= 0)
+	{
+		uint64 limitSegs;
+
+		limitSegs = ConvertToXSegs(max_slot_wal_keep_size_mb, wal_segment_size);
+
+		/* Reduce it if slots already reserves too many. */
+		if (limitSegs < keepSegs)
+			keepSegs = limitSegs;
+	}
+
+	/* but, keep at least wal_keep_segments segments if any */
+	if (wal_keep_segments > 0 && keepSegs < wal_keep_segments)
+		keepSegs = wal_keep_segments;
+
+	/* avoid underflow, don't go below 1 */
+	if (currSeg <= keepSegs)
+		return 1;
+
+	return currSeg - keepSegs;
+}
+
+/*
+ * Calculate remaining bytes until WAL segment for targetLSN will be removed.
+ */
+int64
+DistanceToWalRemoval(XLogRecPtr currLSN, XLogRecPtr targetLSN)
+{
+	XLogSegNo	currSeg;
+	uint64		limitSegs = 0;
+	int64 		restbytes;
+	uint64		fragbytes;
+	XLogSegNo	targetSeg;
+
+	XLByteToSeg(currLSN, currSeg, wal_segment_size);
+
+	/* Calculate how far back WAL segments are preserved */
+	if (max_slot_wal_keep_size_mb >= 0)
+		limitSegs = ConvertToXSegs(max_slot_wal_keep_size_mb, wal_segment_size);
+
+	if (wal_keep_segments > 0 && limitSegs < wal_keep_segments)
+		limitSegs = wal_keep_segments;
+
+	XLByteToSeg(targetLSN, targetSeg, wal_segment_size);
+
+	/* avoid underflow */
+	if (targetSeg + limitSegs < currSeg)
+		return 0;
+
+	/*
+	 * This slot still has all required segments. Calculate how
+	 * many LSN bytes the slot has until it loses targetLSN.
+	 */
+	fragbytes = wal_segment_size - (currLSN % wal_segment_size);
+	XLogSegNoOffsetToRecPtr(targetSeg + limitSegs - currSeg,
+							fragbytes, wal_segment_size,
+							restbytes);
+
+	/*
+	 * Not realistic, but make sure that it is not out of the
+	 * range of int64. No problem to do so since such large values
+	 * have no significant difference.
+	 */
+	if (restbytes > PG_INT64_MAX)
+		restbytes = PG_INT64_MAX;
+
+	return restbytes;
+}
+
 /*
  * Retreat *logSegNo to the last segment that we need to retain because of
  * either wal_keep_segments or replication slots.
@@ -9452,38 +9697,102 @@ CreateRestartPoint(int flags)
 static void
 KeepLogSeg(XLogRecPtr recptr, XLogSegNo *logSegNo)
 {
-	XLogSegNo	segno;
-	XLogRecPtr	keep;
+	static XLogSegNo last_lost_segs = 0;
+	static int last_nslots = 0;
+	static char *last_slot_name = NULL;
+	XLogRecPtr	slotminptr = InvalidXLogRecPtr;
+	XLogSegNo	minSegNo;
+	XLogSegNo	minSlotSegNo;
+	int			nslots_affected = 0;
 
-	XLByteToSeg(recptr, segno, wal_segment_size);
-	keep = XLogGetReplicationSlotMinimumLSN();
+	if (max_replication_slots > 0)
+		slotminptr = XLogGetReplicationSlotMinimumLSN();
 
-	/* compute limit for wal_keep_segments first */
-	if (wal_keep_segments > 0)
+	/*
+	 * We should keep certain number of WAL segments after this checkpoint.
+	 */
+	minSegNo = GetOldestKeepSegment(recptr, slotminptr);
+
+	/*
+	 * Warn the checkpoint is going to flush the segments required by
+	 * replication slots.
+	 */
+	if (!XLogRecPtrIsInvalid(slotminptr))
 	{
-		/* avoid underflow, don't go below 1 */
-		if (segno <= wal_keep_segments)
-			segno = 1;
-		else
-			segno = segno - wal_keep_segments;
+		Assert (max_replication_slots > 0);
+
+		XLByteToSeg(slotminptr, minSlotSegNo, wal_segment_size);
+
+		if (minSlotSegNo < minSegNo)
+		{
+			/* Some slots has lost required segments */
+			XLogSegNo	lost_segs = minSegNo - minSlotSegNo;
+			ReplicationSlot *earliest = NULL;
+			char	   *earliest_name = NULL;
+			int			i;
+
+			/* Find the most affected slot */
+			LWLockAcquire(ReplicationSlotControlLock, LW_SHARED);
+			for (i = 0 ; i < max_replication_slots ; i++)
+			{
+				ReplicationSlot *s =
+					&ReplicationSlotCtl->replication_slots[i];
+				XLogSegNo slotSegNo;
+
+				XLByteToSeg(s->data.restart_lsn, slotSegNo, wal_segment_size);
+
+				if (s->in_use && s->active_pid == 0 && slotSegNo < minSegNo)
+				{
+					nslots_affected++;
+
+					if (earliest == NULL ||
+						s->data.restart_lsn < earliest->data.restart_lsn)
+						earliest = s;
+				}
+			}
+
+			if (earliest)
+			{
+				MemoryContext oldcxt = MemoryContextSwitchTo(TopMemoryContext);
+				earliest_name = pstrdup(NameStr(earliest->data.name));
+				MemoryContextSwitchTo(oldcxt);
+			}
+
+			LWLockRelease(ReplicationSlotControlLock);
+
+			/* Emit WARNING if something has changed */
+			if (earliest_name &&
+				(last_lost_segs != lost_segs || last_nslots != nslots_affected))
+			{
+				ereport(WARNING,
+						(errmsg_plural ("%d replication slot has lost required WAL segments by %lu segments",
+										"%d replication slots have lost required WAL segments by %lu segments",
+										nslots_affected, nslots_affected,
+										lost_segs),
+						 errdetail("Most affected slot is %s.",
+								   earliest_name)));
+
+				if (last_slot_name)
+					pfree(last_slot_name);
+				last_slot_name = earliest_name;
+				last_lost_segs = lost_segs;
+				last_nslots = nslots_affected;
+			}
+		}
 	}
 
-	/* then check whether slots limit removal further */
-	if (max_replication_slots > 0 && keep != InvalidXLogRecPtr)
+	/* Reset the state if no affected slots remain. */
+	if (nslots_affected == 0 && last_slot_name)
 	{
-		XLogSegNo	slotSegNo;
-
-		XLByteToSeg(keep, slotSegNo, wal_segment_size);
-
-		if (slotSegNo <= 0)
-			segno = 1;
-		else if (slotSegNo < segno)
-			segno = slotSegNo;
+		pfree(last_slot_name);
+		last_slot_name = NULL;
+		last_lost_segs = 0;
+		last_nslots = 0;
 	}
 
 	/* don't delete WAL segments newer than the calculated segment */
-	if (segno < *logSegNo)
-		*logSegNo = segno;
+	if (minSegNo < *logSegNo)
+		*logSegNo = minSegNo;
 }
 
 /*
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index 83d00c6cde..775b8b7f20 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -863,7 +863,9 @@ CREATE VIEW pg_replication_slots AS
             L.xmin,
             L.catalog_xmin,
             L.restart_lsn,
-            L.confirmed_flush_lsn
+            L.confirmed_flush_lsn,
+            L.wal_status,
+            L.remain
     FROM pg_get_replication_slots() AS L
             LEFT JOIN pg_database D ON (L.datoid = D.oid);
 
diff --git a/src/backend/replication/slot.c b/src/backend/replication/slot.c
index d90c7235e9..a26f7999aa 100644
--- a/src/backend/replication/slot.c
+++ b/src/backend/replication/slot.c
@@ -49,6 +49,7 @@
 #include "storage/proc.h"
 #include "storage/procarray.h"
 #include "utils/builtins.h"
+#include "utils/memutils.h"
 
 /*
  * Replication slot on-disk data structure.
diff --git a/src/backend/replication/slotfuncs.c b/src/backend/replication/slotfuncs.c
index ce0c9127bc..47cd4375a1 100644
--- a/src/backend/replication/slotfuncs.c
+++ b/src/backend/replication/slotfuncs.c
@@ -234,7 +234,7 @@ pg_drop_replication_slot(PG_FUNCTION_ARGS)
 Datum
 pg_get_replication_slots(PG_FUNCTION_ARGS)
 {
-#define PG_GET_REPLICATION_SLOTS_COLS 11
+#define PG_GET_REPLICATION_SLOTS_COLS 13
 	ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo;
 	TupleDesc	tupdesc;
 	Tuplestorestate *tupstore;
@@ -288,6 +288,7 @@ pg_get_replication_slots(PG_FUNCTION_ARGS)
 		Oid			database;
 		NameData	slot_name;
 		NameData	plugin;
+		WalAvailability walstate;
 		int			i;
 
 		if (!slot->in_use)
@@ -355,6 +356,42 @@ pg_get_replication_slots(PG_FUNCTION_ARGS)
 		else
 			nulls[i++] = true;
 
+		walstate = GetWalAvailability(restart_lsn, active_pid);
+
+		switch (walstate)
+		{
+			case WALAVAIL_INVALID_LSN:
+				nulls[i++] = true;
+				break;
+
+			case WALAVAIL_NORMAL:
+				values[i++] = CStringGetTextDatum("normal");
+				break;
+
+			case WALAVAIL_PRESERVED:
+				values[i++] = CStringGetTextDatum("keeping");
+				break;
+
+			case WALAVAIL_BEING_REMOVED:
+				values[i++] = CStringGetTextDatum("losing");
+				break;
+
+			case WALAVAIL_REMOVED:
+				values[i++] = CStringGetTextDatum("lost");
+				break;
+		}
+
+		if (max_slot_wal_keep_size_mb >= 0 &&
+			(walstate == WALAVAIL_NORMAL ||
+			 walstate == WALAVAIL_PRESERVED))
+		{
+			values[i++] =
+				Int64GetDatum(DistanceToWalRemoval(GetXLogWriteRecPtr(),
+												   restart_lsn));
+		}
+		else
+			nulls[i++] = true;
+
 		tuplestore_putvalues(tupstore, tupdesc, values, nulls);
 	}
 	LWLockRelease(ReplicationSlotControlLock);
diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c
index 79bc7ac8ca..54cd5f6420 100644
--- a/src/backend/utils/misc/guc.c
+++ b/src/backend/utils/misc/guc.c
@@ -2771,6 +2771,19 @@ static struct config_int ConfigureNamesInt[] =
 		NULL, NULL, NULL
 	},
 
+	{
+		{"max_slot_wal_keep_size", PGC_SIGHUP, REPLICATION_SENDING,
+			gettext_noop("Sets the maximum size of WAL space reserved by replication slots."),
+			gettext_noop("Replication slots will be marked as failed, and segments released "
+						 "for deletion or recycling, if this much space is occupied by WAL "
+						 "on disk."),
+			GUC_UNIT_MB
+		},
+		&max_slot_wal_keep_size_mb,
+		-1, -1, MAX_KILOBYTES,
+		NULL, NULL, NULL
+	},
+
 	{
 		{"wal_sender_timeout", PGC_USERSET, REPLICATION_SENDING,
 			gettext_noop("Sets the maximum time to wait for WAL replication."),
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index e9f8ca775d..0b696e7044 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -287,6 +287,7 @@
 #max_wal_senders = 10		# max number of walsender processes
 				# (change requires restart)
 #wal_keep_segments = 0		# in logfile segments; 0 disables
+#max_slot_wal_keep_size = -1	# measured in bytes; -1 disables
 #wal_sender_timeout = 60s	# in milliseconds; 0 disables
 
 #max_replication_slots = 10	# max number of replication slots
diff --git a/src/include/access/xlog.h b/src/include/access/xlog.h
index 9ec7b31cce..c1994eec6f 100644
--- a/src/include/access/xlog.h
+++ b/src/include/access/xlog.h
@@ -108,6 +108,7 @@ extern int	wal_segment_size;
 extern int	min_wal_size_mb;
 extern int	max_wal_size_mb;
 extern int	wal_keep_segments;
+extern int	max_slot_wal_keep_size_mb;
 extern int	XLOGbuffers;
 extern int	XLogArchiveTimeout;
 extern int	wal_retrieve_retry_interval;
@@ -255,6 +256,20 @@ typedef struct CheckpointStatsData
 
 extern CheckpointStatsData CheckpointStats;
 
+/*
+ * WAL segment availability status
+ *
+ * This is used as the return value of GetWalAvailability.
+ */
+typedef enum WalAvailability
+{
+	WALAVAIL_INVALID_LSN,			/* parameter error */
+	WALAVAIL_NORMAL,				/* WAL segment is within max_wal_size */
+	WALAVAIL_PRESERVED,				/* WAL segment is preserved by repslots */
+	WALAVAIL_BEING_REMOVED,			/* WAL segment is no longer preserved */
+	WALAVAIL_REMOVED				/* WAL segment has been removed */
+} WalAvailability;
+
 struct XLogRecData;
 
 extern XLogRecPtr XLogInsertRecord(struct XLogRecData *rdata,
@@ -268,6 +283,7 @@ extern int	XLogFileOpen(XLogSegNo segno);
 
 extern void CheckXLogRemoved(XLogSegNo segno, TimeLineID tli);
 extern XLogSegNo XLogGetLastRemovedSegno(void);
+extern XLogSegNo ScanAndSetLastRemovedSegno(void);
 extern void XLogSetAsyncXactLSN(XLogRecPtr record);
 extern void XLogSetReplicationSlotMinimumLSN(XLogRecPtr lsn);
 
@@ -305,6 +321,9 @@ extern void ShutdownXLOG(int code, Datum arg);
 extern void InitXLOGAccess(void);
 extern void CreateCheckPoint(int flags);
 extern bool CreateRestartPoint(int flags);
+extern WalAvailability GetWalAvailability(XLogRecPtr restart_lsn,
+										  pid_t walsender_pid);
+extern int64 DistanceToWalRemoval(XLogRecPtr currLSN, XLogRecPtr targetLSN);
 extern void XLogPutNextOid(Oid nextOid);
 extern XLogRecPtr XLogRestorePoint(const char *rpName);
 extern void UpdateFullPageWrites(void);
diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat
index a6a708cca9..2025f34bfd 100644
--- a/src/include/catalog/pg_proc.dat
+++ b/src/include/catalog/pg_proc.dat
@@ -9971,9 +9971,9 @@
   proname => 'pg_get_replication_slots', prorows => '10', proisstrict => 'f',
   proretset => 't', provolatile => 's', prorettype => 'record',
   proargtypes => '',
-  proallargtypes => '{name,name,text,oid,bool,bool,int4,xid,xid,pg_lsn,pg_lsn}',
-  proargmodes => '{o,o,o,o,o,o,o,o,o,o,o}',
-  proargnames => '{slot_name,plugin,slot_type,datoid,temporary,active,active_pid,xmin,catalog_xmin,restart_lsn,confirmed_flush_lsn}',
+  proallargtypes => '{name,name,text,oid,bool,bool,int4,xid,xid,pg_lsn,pg_lsn,text,int8}',
+  proargmodes => '{o,o,o,o,o,o,o,o,o,o,o,o,o}',
+  proargnames => '{slot_name,plugin,slot_type,datoid,temporary,active,active_pid,xmin,catalog_xmin,restart_lsn,confirmed_flush_lsn,wal_status,remain}',
   prosrc => 'pg_get_replication_slots' },
 { oid => '3786', descr => 'set up a logical replication slot',
   proname => 'pg_create_logical_replication_slot', provolatile => 'v',
diff --git a/src/test/recovery/t/018_replslot_limit.pl b/src/test/recovery/t/018_replslot_limit.pl
new file mode 100644
index 0000000000..84a1f3a9dd
--- /dev/null
+++ b/src/test/recovery/t/018_replslot_limit.pl
@@ -0,0 +1,202 @@
+# Test for replication slot limit
+# Ensure that max_slot_wal_keep_size limits the number of WAL files to
+# be kept by replication slots.
+
+use strict;
+use warnings;
+use File::Path qw(rmtree);
+use PostgresNode;
+use TestLib;
+use Test::More tests => 13;
+use Time::HiRes qw(usleep);
+
+$ENV{PGDATABASE} = 'postgres';
+
+# Initialize master node, setting wal-segsize to 1MB
+my $node_master = get_new_node('master');
+$node_master->init(allows_streaming => 1,
+				   extra => ['--wal-segsize=1'],
+				   min_wal_size => '2MB',
+				   max_wal_size => '4MB',
+				   log_checkpoints => 'yes');
+$node_master->start;
+$node_master->safe_psql('postgres', "SELECT pg_create_physical_replication_slot('rep1')");
+
+# The slot state and remain should be null before the first connection
+my $result = $node_master->safe_psql('postgres', "SELECT restart_lsn is NULL, wal_status is NULL, remain is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'");
+is($result, "t|t|t", 'check the state of non-reserved slot is "unknown"');
+
+
+# Take backup
+my $backup_name = 'my_backup';
+$node_master->backup($backup_name);
+
+# Create a standby linking to it using the replication slot
+my $node_standby = get_new_node('standby_1');
+$node_standby->init_from_backup($node_master, $backup_name, has_streaming => 1,
+								primary_slot_name => 'rep1');
+
+$node_standby->start;
+
+# Wait until standby has replayed enough data
+my $start_lsn = $node_master->lsn('write');
+$node_master->wait_for_catchup($node_standby, 'replay', $start_lsn);
+
+# Stop standby
+$node_standby->stop;
+
+
+# Preparation done, the slot is the state "normal" now
+$result = $node_master->safe_psql('postgres', "SELECT restart_lsn, wal_status, remain is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'");
+is($result, "$start_lsn|normal|t", 'check the catching-up state');
+
+# Advance WAL by five segments (= 5MB) on master
+advance_wal($node_master, 1);
+$node_master->safe_psql('postgres', "CHECKPOINT;");
+
+# The slot is always "safe" when fitting max_wal_size
+$result = $node_master->safe_psql('postgres', "SELECT restart_lsn, wal_status, remain is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'");
+is($result, "$start_lsn|normal|t", 'check that restart_lsn is in max_wal_size');
+
+advance_wal($node_master, 4);
+$node_master->safe_psql('postgres', "CHECKPOINT;");
+
+# The slot is always "safe" when max_slot_wal_keep_size is not set
+$result = $node_master->safe_psql('postgres', "SELECT restart_lsn, wal_status, remain is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'");
+is($result, "$start_lsn|normal|t", 'check that slot is working');
+
+# The standby can reconnect to master
+$node_standby->start;
+
+$start_lsn = $node_master->lsn('write');
+$node_master->wait_for_catchup($node_standby, 'replay', $start_lsn);
+
+$node_standby->stop;
+
+# Set max_slot_wal_keep_size on master
+my $max_slot_wal_keep_size_mb = 6;
+$node_master->append_conf('postgresql.conf', qq(
+max_slot_wal_keep_size = ${max_slot_wal_keep_size_mb}MB
+));
+$node_master->reload;
+
+# The slot is in safe state. The remaining bytes should be as almost
+# (max_slot_wal_keep_size + 1) times large as the segment size
+$result = $node_master->safe_psql('postgres', "SELECT restart_lsn, wal_status, pg_size_pretty(remain) as remain FROM pg_replication_slots WHERE slot_name = 'rep1'");
+is($result, "$start_lsn|normal|7168 kB", 'check that max_slot_wal_keep_size is working');
+
+# Advance WAL again then checkpoint, reducing remain by 2 MB.
+advance_wal($node_master, 2);
+$node_master->safe_psql('postgres', "CHECKPOINT;");
+
+# The slot is still working
+$result = $node_master->safe_psql('postgres', "SELECT restart_lsn, wal_status, pg_size_pretty(remain) as remain FROM pg_replication_slots WHERE slot_name = 'rep1'");
+is($result, "$start_lsn|normal|5120 kB", 'check that remaining byte is calculated correctly');
+
+# wal_keep_segments overrides max_slot_wal_keep_size
+$result = $node_master->safe_psql('postgres', "ALTER SYSTEM SET wal_keep_segments to 8; SELECT pg_reload_conf();");
+$result = $node_master->safe_psql('postgres', "SELECT restart_lsn, wal_status, pg_size_pretty(remain) as remain FROM pg_replication_slots WHERE slot_name = 'rep1'");
+is($result, "$start_lsn|normal|7168 kB", 'check that wal_keep_segments overrides max_slot_wal_keep_size');
+
+# restore wal_keep_segments
+$result = $node_master->safe_psql('postgres', "ALTER SYSTEM SET wal_keep_segments to 0; SELECT pg_reload_conf();");
+
+# Advance WAL again without checkpoint, reducing remain by 4 MB.
+advance_wal($node_master, 4);
+
+# Slot gets into 'keeping' state
+$result = $node_master->safe_psql('postgres', "SELECT restart_lsn, wal_status, pg_size_pretty(remain) as remain FROM pg_replication_slots WHERE slot_name = 'rep1'");
+is($result, "$start_lsn|keeping|1024 kB", 'check that the slot state changes to "keeping"');
+
+# do checkpoint so that the next checkpoint runs too early
+$node_master->safe_psql('postgres', "CHECKPOINT;");
+
+# Advance WAL again without checkpoint; remain goes to 0.
+advance_wal($node_master, 1);
+
+# Slot gets into 'lost' state
+$result = $node_master->safe_psql('postgres', "SELECT restart_lsn, wal_status, remain is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'");
+is($result, "$start_lsn|lost|t", 'check that the slot state changes to "lost"');
+
+# The standby still can connect to master before a checkpoint
+$node_standby->start;
+
+$start_lsn = $node_master->lsn('write');
+$node_master->wait_for_catchup($node_standby, 'replay', $start_lsn);
+
+$node_standby->stop;
+
+ok(!find_in_log($node_standby,
+				"requested WAL segment [0-9A-F]+ has already been removed"),
+   'check that required WAL segments are still available');
+
+# Advance WAL again, the slot loses the oldest segment.
+my $logstart = get_log_size($node_master);
+advance_wal($node_master, 7);
+$node_master->safe_psql('postgres', "CHECKPOINT;");
+
+# WARNING should be issued
+ok(find_in_log($node_master,
+			   "1 replication slot has lost required WAL segments by 1 segments\n".
+			   ".*Most affected slot is rep1.",
+			   $logstart),
+   'check that the warning is logged');
+
+# This slot should be broken
+$result = $node_master->safe_psql('postgres', "SELECT restart_lsn, wal_status, remain is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'");
+is($result, "$start_lsn|lost|t", 'check that the slot state changes to "lost"');
+
+# The standby no longer can connect to the master
+$logstart = get_log_size($node_standby);
+$node_standby->start;
+
+my $failed = 0;
+for (my $i = 0 ; $i < 10000 ; $i++)
+{
+	if (find_in_log($node_standby,
+					"requested WAL segment [0-9A-F]+ has already been removed",
+					$logstart))
+	{
+		$failed = 1;
+		last;
+	}
+	usleep(100_000);
+}
+ok($failed, 'check that replication has been broken');
+
+$node_standby->stop;
+
+#####################################
+# Advance WAL of $node by $n segments
+sub advance_wal
+{
+	my ($node, $n) = @_;
+
+	# Advance by $n segments (= (16 * $n) MB) on master
+	for (my $i = 0 ; $i < $n ; $i++)
+	{
+		$node->safe_psql('postgres', "CREATE TABLE t (); DROP TABLE t; SELECT pg_switch_wal();");
+	}
+}
+
+# return the size of logfile of $node in bytes
+sub get_log_size
+{
+	my ($node) = @_;
+
+	return (stat $node->logfile)[7];
+}
+
+# find $pat in logfile of $node after $off-th byte
+sub find_in_log
+{
+	my ($node, $pat, $off) = @_;
+
+	$off = 0 unless defined $off;
+	my $log = TestLib::slurp_file($node->logfile);
+	return 0 if (length($log) <= $off);
+
+	$log = substr($log, $off);
+
+	return $log =~ m/$pat/;
+}
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index 7245b0e13b..8688f7138f 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1462,8 +1462,10 @@ pg_replication_slots| SELECT l.slot_name,
     l.xmin,
     l.catalog_xmin,
     l.restart_lsn,
-    l.confirmed_flush_lsn
-   FROM (pg_get_replication_slots() l(slot_name, plugin, slot_type, datoid, temporary, active, active_pid, xmin, catalog_xmin, restart_lsn, confirmed_flush_lsn)
+    l.confirmed_flush_lsn,
+    l.wal_status,
+    l.remain
+   FROM (pg_get_replication_slots() l(slot_name, plugin, slot_type, datoid, temporary, active, active_pid, xmin, catalog_xmin, restart_lsn, confirmed_flush_lsn, wal_status, remain)
      LEFT JOIN pg_database d ON ((l.datoid = d.oid)));
 pg_roles| SELECT pg_authid.rolname,
     pg_authid.rolsuper,
-- 
2.18.2


----Next_Part(Tue_Mar_31_14_10_14_2020_637)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v20-0003-Allow-init-and-init_from_backup-to-set-arbitrary.patch"



^ permalink  raw  reply  [nested|flat] 125+ messages in thread

* [PATCH 1/6] Add WAL relief vent for replication slots
@ 2017-12-21 12:20 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 125+ messages in thread

From: Kyotaro Horiguchi @ 2017-12-21 12:20 UTC (permalink / raw)

Replication slot is useful to maintain replication connection in the
configurations where replication is so delayed that connection is
broken. On the other hand so many WAL files can fill up disk that the
master downs by a long delay. This feature, which is activated by a
GUC "max_slot_wal_keep_size", protects master servers from suffering
disk full by limiting the number of WAL files reserved by replication
slots.
---
 src/backend/access/transam/xlog.c             | 124 ++++++++++++++----
 src/backend/replication/slot.c                |  57 ++++++++
 src/backend/utils/misc/guc.c                  |  12 ++
 src/backend/utils/misc/postgresql.conf.sample |   1 +
 src/include/access/xlog.h                     |   1 +
 src/include/replication/slot.h                |   1 +
 6 files changed, 173 insertions(+), 23 deletions(-)

diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index e08320e829..9bea2a403f 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -104,6 +104,7 @@ int			wal_level = WAL_LEVEL_MINIMAL;
 int			CommitDelay = 0;	/* precommit delay in microseconds */
 int			CommitSiblings = 5; /* # concurrent xacts needed to sleep */
 int			wal_retrieve_retry_interval = 5000;
+int			max_slot_wal_keep_size_mb = -1;
 
 #ifdef WAL_DEBUG
 bool		XLOG_DEBUG = false;
@@ -871,6 +872,7 @@ static void checkTimeLineSwitch(XLogRecPtr lsn, TimeLineID newTLI,
 static void LocalSetXLogInsertAllowed(void);
 static void CreateEndOfRecoveryRecord(void);
 static void CheckPointGuts(XLogRecPtr checkPointRedo, int flags);
+static XLogSegNo GetOldestKeepSegment(XLogRecPtr currpos, XLogRecPtr minSlotPtr);
 static void KeepLogSeg(XLogRecPtr recptr, XLogSegNo *logSegNo);
 static XLogRecPtr XLogGetReplicationSlotMinimumLSN(void);
 
@@ -9272,6 +9274,54 @@ CreateRestartPoint(int flags)
 	return true;
 }
 
+/*
+ * Returns minimum segment number that the next checkpoint must leave
+ * considering wal_keep_segments, replication slots and
+ * max_slot_wal_keep_size.
+ *
+ * currLSN is the current insert location.
+ * minSlotLSN is the minimum restart_lsn of all active slots.
+ */
+static XLogSegNo
+GetOldestKeepSegment(XLogRecPtr currLSN, XLogRecPtr minSlotLSN)
+{
+	XLogSegNo	currSeg;
+	XLogSegNo	minSlotSeg;
+	uint64		keepSegs = 0;	/* # of segments actually kept */
+
+	XLByteToSeg(currLSN, currSeg, wal_segment_size);
+	XLByteToSeg(minSlotLSN, minSlotSeg, wal_segment_size);
+
+	/*
+	 * Calculate how many segments are kept by slots first. The second
+	 * term of the condition is just a sanity check.
+	 */
+	if (minSlotLSN != InvalidXLogRecPtr && minSlotSeg <= currSeg)
+		keepSegs = currSeg - minSlotSeg;
+
+	/* Cap keepSegs by max_slot_wal_keep_size */
+	if (max_slot_wal_keep_size_mb >= 0)
+	{
+		uint64 limitSegs;
+
+		limitSegs = ConvertToXSegs(max_slot_wal_keep_size_mb, wal_segment_size);
+
+		/* Reduce it if slots already reserves too many. */
+		if (limitSegs < keepSegs)
+			keepSegs = limitSegs;
+	}
+
+	/* but, keep at least wal_keep_segments segments if any */
+	if (wal_keep_segments > 0 && keepSegs < wal_keep_segments)
+		keepSegs = wal_keep_segments;
+
+	/* avoid underflow, don't go below 1 */
+	if (currSeg <= keepSegs)
+		return 1;
+
+	return currSeg - keepSegs;
+}
+
 /*
  * Retreat *logSegNo to the last segment that we need to retain because of
  * either wal_keep_segments or replication slots.
@@ -9283,38 +9333,66 @@ CreateRestartPoint(int flags)
 static void
 KeepLogSeg(XLogRecPtr recptr, XLogSegNo *logSegNo)
 {
-	XLogSegNo	segno;
-	XLogRecPtr	keep;
+	XLogRecPtr	slotminptr = InvalidXLogRecPtr;
+	XLogSegNo	minSegNo;
+	XLogSegNo	slotSegNo;
 
-	XLByteToSeg(recptr, segno, wal_segment_size);
-	keep = XLogGetReplicationSlotMinimumLSN();
+	if (max_replication_slots > 0)
+		slotminptr = XLogGetReplicationSlotMinimumLSN();
 
-	/* compute limit for wal_keep_segments first */
-	if (wal_keep_segments > 0)
-	{
-		/* avoid underflow, don't go below 1 */
-		if (segno <= wal_keep_segments)
-			segno = 1;
-		else
-			segno = segno - wal_keep_segments;
-	}
+	/*
+	 * We should keep certain number of WAL segments after this checkpoint.
+	 */
+	minSegNo = GetOldestKeepSegment(recptr, slotminptr);
 
-	/* then check whether slots limit removal further */
-	if (max_replication_slots > 0 && keep != InvalidXLogRecPtr)
+	/*
+	 * Warn the checkpoint is going to flush the segments required by
+	 * replication slots.
+	 */
+	if (!XLogRecPtrIsInvalid(slotminptr))
 	{
-		XLogSegNo	slotSegNo;
+		static XLogSegNo prev_lost_segs = 0;	/* avoid duplicate messages */
 
-		XLByteToSeg(keep, slotSegNo, wal_segment_size);
+		XLByteToSeg(slotminptr, slotSegNo, wal_segment_size);
 
-		if (slotSegNo <= 0)
-			segno = 1;
-		else if (slotSegNo < segno)
-			segno = slotSegNo;
+		if (slotSegNo < minSegNo)
+		{
+			XLogSegNo lost_segs = minSegNo - slotSegNo;
+			if (prev_lost_segs != lost_segs)
+			{
+				/* We have lost a new segment, warn it.*/
+				XLogRecPtr minlsn;
+				char *slot_names;
+				int nslots;
+
+				XLogSegNoOffsetToRecPtr(minSegNo, 0, wal_segment_size, minlsn);
+				slot_names =
+					ReplicationSlotsEnumerateBehinds(minlsn, ", ", &nslots);
+
+				/*
+				 * Some of the affected slots could have just been removed.
+				 * We don't need show anything here if no affected slot
+				 * remains.
+				 */
+				if (slot_names)
+				{
+					ereport(WARNING,
+							(errmsg ("some replication slots have lost required WAL segments"),
+							 errdetail_plural(
+								 "Slot %s lost %ld segment(s).",
+								 "Slots %s lost at most %ld segment(s).",
+								 nslots, slot_names, lost_segs)));
+				}
+			}
+			prev_lost_segs = lost_segs;
+		}
+		else
+			prev_lost_segs = 0;
 	}
 
 	/* don't delete WAL segments newer than the calculated segment */
-	if (segno < *logSegNo)
-		*logSegNo = segno;
+	if (minSegNo < *logSegNo)
+		*logSegNo = minSegNo;
 }
 
 /*
diff --git a/src/backend/replication/slot.c b/src/backend/replication/slot.c
index 55c306e465..7f9eab6edf 100644
--- a/src/backend/replication/slot.c
+++ b/src/backend/replication/slot.c
@@ -1064,6 +1064,63 @@ ReplicationSlotReserveWal(void)
 	}
 }
 
+/*
+ * Returns names of replication slots that their restart_lsn are behind
+ * specified LSN, in palloc'ed character array stuffed with slot names
+ * delimited by the given separator.  Returns NULL if no slot matches. If
+ * pnslots is given, the number of the returned slots is returned there.
+ */
+char *
+ReplicationSlotsEnumerateBehinds(XLogRecPtr target, char *separator, int *pnslots)
+{
+	static StringInfoData retstr;
+	static bool retstr_initialized = false;
+	bool insert_separator = false;
+	int i;
+	int nslots = 0;
+
+	Assert (separator);
+	if (max_replication_slots <= 0)
+		return NULL;
+
+	if (!retstr_initialized)
+	{
+		initStringInfo(&retstr);
+		retstr_initialized = true;
+	}
+	else
+		resetStringInfo(&retstr);
+
+	/* construct name list */
+	LWLockAcquire(ReplicationSlotControlLock, LW_SHARED);
+	for (i = 0 ; i < max_replication_slots ; i++)
+	{
+		ReplicationSlot *s = &ReplicationSlotCtl->replication_slots[i];
+
+		if (s->in_use && s->data.restart_lsn < target)
+		{
+			if (insert_separator)
+				appendStringInfoString(&retstr, separator);
+
+			/*
+			 * Slot names consist only with lower-case letters. We don't
+			 * bother quoting.
+			 */
+			appendStringInfoString(&retstr, NameStr(s->data.name));
+			insert_separator = true;
+			nslots++;
+		}
+	}
+	LWLockRelease(ReplicationSlotControlLock);
+
+	/* return the number of slots in the list if requested */
+	if (pnslots)
+		*pnslots = nslots;
+
+	/* return NULL instead of an empty string */
+	return retstr.data[0] ? retstr.data : NULL;
+}
+
 /*
  * Flush all replication slots to disk.
  *
diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c
index 1208eb9a68..80245e9def 100644
--- a/src/backend/utils/misc/guc.c
+++ b/src/backend/utils/misc/guc.c
@@ -2655,6 +2655,18 @@ static struct config_int ConfigureNamesInt[] =
 		NULL, NULL, NULL
 	},
 
+	{
+		{"max_slot_wal_keep_size", PGC_SIGHUP, REPLICATION_SENDING,
+			gettext_noop("Sets the maximum size of extra WALs kept by replication slots."),
+		 NULL,
+		 GUC_UNIT_MB
+		},
+		&max_slot_wal_keep_size_mb,
+		-1, -1,
+		MAX_KILOBYTES, /* XXX: This is in megabytes, like max/min_wal_size */
+		NULL, NULL, NULL
+	},
+
 	{
 		{"wal_sender_timeout", PGC_USERSET, REPLICATION_SENDING,
 			gettext_noop("Sets the maximum time to wait for WAL replication."),
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index 5ee5e09ddf..5da2706a99 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -286,6 +286,7 @@
 #max_wal_senders = 10		# max number of walsender processes
 				# (change requires restart)
 #wal_keep_segments = 0		# in logfile segments; 0 disables
+#max_slot_wal_keep_size = -1	# measured in bytes; -1 disables
 #wal_sender_timeout = 60s	# in milliseconds; 0 disables
 
 #max_replication_slots = 10	# max number of replication slots
diff --git a/src/include/access/xlog.h b/src/include/access/xlog.h
index 237f4e0350..e5322abdf5 100644
--- a/src/include/access/xlog.h
+++ b/src/include/access/xlog.h
@@ -108,6 +108,7 @@ extern int	wal_segment_size;
 extern int	min_wal_size_mb;
 extern int	max_wal_size_mb;
 extern int	wal_keep_segments;
+extern int	max_slot_wal_keep_size_mb;
 extern int	XLOGbuffers;
 extern int	XLogArchiveTimeout;
 extern int	wal_retrieve_retry_interval;
diff --git a/src/include/replication/slot.h b/src/include/replication/slot.h
index 8fbddea78f..e0fee0663c 100644
--- a/src/include/replication/slot.h
+++ b/src/include/replication/slot.h
@@ -199,6 +199,7 @@ extern void ReplicationSlotsComputeRequiredLSN(void);
 extern XLogRecPtr ReplicationSlotsComputeLogicalRestartLSN(void);
 extern bool ReplicationSlotsCountDBSlots(Oid dboid, int *nslots, int *nactive);
 extern void ReplicationSlotsDropDBSlots(Oid dboid);
+extern char *ReplicationSlotsEnumerateBehinds(XLogRecPtr target, char *separator, int *pnslots);
 
 extern void StartupReplicationSlots(void);
 extern void CheckPointReplicationSlots(void);
-- 
2.20.1


--MP_/68vU=6Ib.yoYOTdA.M=EVne
Content-Type: text/x-patch
Content-Transfer-Encoding: 7bit
Content-Disposition: attachment;
 filename=v14-0002-Add-monitoring-aid-for-max_slot_wal_keep_size.patch



^ permalink  raw  reply  [nested|flat] 125+ messages in thread

* [PATCH 1/6] Add WAL relief vent for replication slots
@ 2017-12-21 12:20 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 125+ messages in thread

From: Kyotaro Horiguchi @ 2017-12-21 12:20 UTC (permalink / raw)

Replication slot is useful to maintain replication connection in the
configurations where replication is so delayed that connection is
broken. On the other hand so many WAL files can fill up disk that the
master downs by a long delay. This feature, which is activated by a
GUC "max_slot_wal_keep_size", protects master servers from suffering
disk full by limiting the number of WAL files reserved by replication
slots.
---
 src/backend/access/transam/xlog.c             | 128 +++++++++++++++++++++-----
 src/backend/replication/slot.c                |  57 ++++++++++++
 src/backend/utils/misc/guc.c                  |  12 +++
 src/backend/utils/misc/postgresql.conf.sample |   1 +
 src/include/access/xlog.h                     |   1 +
 src/include/replication/slot.h                |   1 +
 6 files changed, 175 insertions(+), 25 deletions(-)

diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index ecd12fc53a..998b779277 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -100,6 +100,7 @@ int			wal_level = WAL_LEVEL_MINIMAL;
 int			CommitDelay = 0;	/* precommit delay in microseconds */
 int			CommitSiblings = 5; /* # concurrent xacts needed to sleep */
 int			wal_retrieve_retry_interval = 5000;
+int			max_slot_wal_keep_size_mb = -1;
 
 #ifdef WAL_DEBUG
 bool		XLOG_DEBUG = false;
@@ -872,6 +873,7 @@ static void checkTimeLineSwitch(XLogRecPtr lsn, TimeLineID newTLI,
 static void LocalSetXLogInsertAllowed(void);
 static void CreateEndOfRecoveryRecord(void);
 static void CheckPointGuts(XLogRecPtr checkPointRedo, int flags);
+static XLogSegNo GetOldestKeepSegment(XLogRecPtr currpos, XLogRecPtr minSlotPtr);
 static void KeepLogSeg(XLogRecPtr recptr, XLogSegNo *logSegNo);
 static XLogRecPtr XLogGetReplicationSlotMinimumLSN(void);
 
@@ -9329,6 +9331,54 @@ CreateRestartPoint(int flags)
 	return true;
 }
 
+/*
+ * Returns minimum segment number that the next checkpoint must leave
+ * considering wal_keep_segments, replication slots and
+ * max_slot_wal_keep_size.
+ *
+ * currLSN is the current insert location.
+ * minSlotLSN is the minimum restart_lsn of all active slots.
+ */
+static XLogSegNo
+GetOldestKeepSegment(XLogRecPtr currLSN, XLogRecPtr minSlotLSN)
+{
+	XLogSegNo	currSeg;
+	XLogSegNo	minSlotSeg;
+	uint64		keepSegs = 0;	/* # of segments actually kept */
+
+	XLByteToSeg(currLSN, currSeg, wal_segment_size);
+	XLByteToSeg(minSlotLSN, minSlotSeg, wal_segment_size);
+
+	/*
+	 * Calculate how many segments are kept by slots first. The second
+	 * term of the condition is just a sanity check.
+	 */
+	if (minSlotLSN != InvalidXLogRecPtr && minSlotSeg <= currSeg)
+		keepSegs = currSeg - minSlotSeg;
+
+	/* Cap keepSegs by max_slot_wal_keep_size */
+	if (max_slot_wal_keep_size_mb >= 0)
+	{
+		uint64 limitSegs;
+
+		limitSegs = ConvertToXSegs(max_slot_wal_keep_size_mb, wal_segment_size);
+
+		/* Reduce it if slots already reserves too many. */
+		if (limitSegs < keepSegs)
+			keepSegs = limitSegs;
+	}
+
+	/* but, keep at least wal_keep_segments segments if any */
+	if (wal_keep_segments > 0 && keepSegs < wal_keep_segments)
+		keepSegs = wal_keep_segments;
+
+	/* avoid underflow, don't go below 1 */
+	if (currSeg <= keepSegs)
+		return 1;
+
+	return currSeg - keepSegs;
+}
+
 /*
  * Retreat *logSegNo to the last segment that we need to retain because of
  * either wal_keep_segments or replication slots.
@@ -9340,38 +9390,66 @@ CreateRestartPoint(int flags)
 static void
 KeepLogSeg(XLogRecPtr recptr, XLogSegNo *logSegNo)
 {
-	XLogSegNo	segno;
-	XLogRecPtr	keep;
+	XLogRecPtr	slotminptr = InvalidXLogRecPtr;
+	XLogSegNo	minSegNo;
+	XLogSegNo	slotSegNo;
 
-	XLByteToSeg(recptr, segno, wal_segment_size);
-	keep = XLogGetReplicationSlotMinimumLSN();
+	if (max_replication_slots > 0)
+		slotminptr = XLogGetReplicationSlotMinimumLSN();
 
-	/* compute limit for wal_keep_segments first */
-	if (wal_keep_segments > 0)
+	/*
+	 * We should keep certain number of WAL segments after this checkpoint.
+	 */
+	minSegNo = GetOldestKeepSegment(recptr, slotminptr);
+
+	/*
+	 * Warn the checkpoint is going to flush the segments required by
+	 * replication slots.
+	 */
+	if (!XLogRecPtrIsInvalid(slotminptr))
 	{
-		/* avoid underflow, don't go below 1 */
-		if (segno <= wal_keep_segments)
-			segno = 1;
+		static XLogSegNo prev_lost_segs = 0;	/* avoid duplicate messages */
+
+		XLByteToSeg(slotminptr, slotSegNo, wal_segment_size);
+
+		if (slotSegNo < minSegNo)
+		{
+			XLogSegNo lost_segs = minSegNo - slotSegNo;
+			if (prev_lost_segs != lost_segs)
+			{
+				/* We have lost a new segment, warn it.*/
+				XLogRecPtr minlsn;
+				char *slot_names;
+				int nslots;
+
+				XLogSegNoOffsetToRecPtr(minSegNo, 0, wal_segment_size, minlsn);
+				slot_names =
+					ReplicationSlotsEnumerateBehinds(minlsn, ", ", &nslots);
+
+				/*
+				 * Some of the affected slots could have just been removed.
+				 * We don't need show anything here if no affected slot
+				 * remains.
+				 */
+				if (slot_names)
+				{
+					ereport(WARNING,
+							(errmsg ("some replication slots have lost required WAL segments"),
+							 errdetail_plural(
+								 "Slot %s lost %ld segment(s).",
+								 "Slots %s lost at most %ld segment(s).",
+								 nslots, slot_names, lost_segs)));
+				}
+			}
+			prev_lost_segs = lost_segs;
+		}
 		else
-			segno = segno - wal_keep_segments;
-	}
-
-	/* then check whether slots limit removal further */
-	if (max_replication_slots > 0 && keep != InvalidXLogRecPtr)
-	{
-		XLogSegNo	slotSegNo;
-
-		XLByteToSeg(keep, slotSegNo, wal_segment_size);
-
-		if (slotSegNo <= 0)
-			segno = 1;
-		else if (slotSegNo < segno)
-			segno = slotSegNo;
+			prev_lost_segs = 0;
 	}
 
 	/* don't delete WAL segments newer than the calculated segment */
-	if (segno < *logSegNo)
-		*logSegNo = segno;
+	if (minSegNo < *logSegNo)
+		*logSegNo = minSegNo;
 }
 
 /*
diff --git a/src/backend/replication/slot.c b/src/backend/replication/slot.c
index 33b23b6b6d..6ef63ae7c0 100644
--- a/src/backend/replication/slot.c
+++ b/src/backend/replication/slot.c
@@ -1064,6 +1064,63 @@ ReplicationSlotReserveWal(void)
 	}
 }
 
+/*
+ * Returns names of replication slots that their restart_lsn are behind
+ * specified LSN, in palloc'ed character array stuffed with slot names
+ * delimited by the given separator.  Returns NULL if no slot matches. If
+ * pnslots is given, the number of the returned slots is returned there.
+ */
+char *
+ReplicationSlotsEnumerateBehinds(XLogRecPtr target, char *separator, int *pnslots)
+{
+	static StringInfoData retstr;
+	static bool retstr_initialized = false;
+	bool insert_separator = false;
+	int i;
+	int nslots = 0;
+
+	Assert (separator);
+	if (max_replication_slots <= 0)
+		return NULL;
+
+	if (!retstr_initialized)
+	{
+		initStringInfo(&retstr);
+		retstr_initialized = true;
+	}
+	else
+		resetStringInfo(&retstr);
+
+	/* construct name list */
+	LWLockAcquire(ReplicationSlotControlLock, LW_SHARED);
+	for (i = 0 ; i < max_replication_slots ; i++)
+	{
+		ReplicationSlot *s = &ReplicationSlotCtl->replication_slots[i];
+
+		if (s->in_use && s->data.restart_lsn < target)
+		{
+			if (insert_separator)
+				appendStringInfoString(&retstr, separator);
+
+			/*
+			 * Slot names consist only with lower-case letters. We don't
+			 * bother quoting.
+			 */
+			appendStringInfoString(&retstr, NameStr(s->data.name));
+			insert_separator = true;
+			nslots++;
+		}
+	}
+	LWLockRelease(ReplicationSlotControlLock);
+
+	/* return the number of slots in the list if requested */
+	if (pnslots)
+		*pnslots = nslots;
+
+	/* return NULL instead of an empty string */
+	return retstr.data[0] ? retstr.data : NULL;
+}
+
 /*
  * Flush all replication slots to disk.
  *
diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c
index 156d147c85..c5f04fb8a5 100644
--- a/src/backend/utils/misc/guc.c
+++ b/src/backend/utils/misc/guc.c
@@ -2629,6 +2629,18 @@ static struct config_int ConfigureNamesInt[] =
 		NULL, NULL, NULL
 	},
 
+	{
+		{"max_slot_wal_keep_size", PGC_SIGHUP, REPLICATION_SENDING,
+			gettext_noop("Sets the maximum size of extra WALs kept by replication slots."),
+		 NULL,
+		 GUC_UNIT_MB
+		},
+		&max_slot_wal_keep_size_mb,
+		-1, -1,
+		MAX_KILOBYTES, /* XXX: This is in megabytes, like max/min_wal_size */
+		NULL, NULL, NULL
+	},
+
 	{
 		{"wal_sender_timeout", PGC_USERSET, REPLICATION_SENDING,
 			gettext_noop("Sets the maximum time to wait for WAL replication."),
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index 194f312096..6f96177bbd 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -282,6 +282,7 @@
 #max_wal_senders = 10		# max number of walsender processes
 				# (change requires restart)
 #wal_keep_segments = 0		# in logfile segments; 0 disables
+#max_slot_wal_keep_size = -1	# measured in bytes; -1 disables
 #wal_sender_timeout = 60s	# in milliseconds; 0 disables
 
 #max_replication_slots = 10	# max number of replication slots
diff --git a/src/include/access/xlog.h b/src/include/access/xlog.h
index f90a6a9139..b2eb30b779 100644
--- a/src/include/access/xlog.h
+++ b/src/include/access/xlog.h
@@ -108,6 +108,7 @@ extern int	wal_segment_size;
 extern int	min_wal_size_mb;
 extern int	max_wal_size_mb;
 extern int	wal_keep_segments;
+extern int	max_slot_wal_keep_size_mb;
 extern int	XLOGbuffers;
 extern int	XLogArchiveTimeout;
 extern int	wal_retrieve_retry_interval;
diff --git a/src/include/replication/slot.h b/src/include/replication/slot.h
index a8f1d66bae..9c3635dc0e 100644
--- a/src/include/replication/slot.h
+++ b/src/include/replication/slot.h
@@ -199,6 +199,7 @@ extern void ReplicationSlotsComputeRequiredLSN(void);
 extern XLogRecPtr ReplicationSlotsComputeLogicalRestartLSN(void);
 extern bool ReplicationSlotsCountDBSlots(Oid dboid, int *nslots, int *nactive);
 extern void ReplicationSlotsDropDBSlots(Oid dboid);
+extern char *ReplicationSlotsEnumerateBehinds(XLogRecPtr target, char *separator, int *pnslots);
 
 extern void StartupReplicationSlots(void);
 extern void CheckPointReplicationSlots(void);
-- 
2.16.3


----Next_Part(Fri_Feb_22_14_12_28_2019_076)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v13-0002-Add-monitoring-aid-for-max_slot_wal_keep_size.patch"



^ permalink  raw  reply  [nested|flat] 125+ messages in thread

* [PATCH 1/6] Add WAL relief vent for replication slots
@ 2017-12-21 12:20 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 125+ messages in thread

From: Kyotaro Horiguchi @ 2017-12-21 12:20 UTC (permalink / raw)

Replication slot is useful to maintain replication connection in the
configurations where replication is so delayed that connection is
broken. On the other hand so many WAL files can fill up disk that the
master downs by a long delay. This feature, which is activated by a
GUC "max_slot_wal_keep_size", protects master servers from suffering
disk full by limiting the number of WAL files reserved by replication
slots.
---
 src/backend/access/transam/xlog.c             | 128 +++++++++++++++++++++-----
 src/backend/replication/slot.c                |  62 +++++++++++++
 src/backend/utils/misc/guc.c                  |  12 +++
 src/backend/utils/misc/postgresql.conf.sample |   1 +
 src/include/access/xlog.h                     |   1 +
 src/include/replication/slot.h                |   1 +
 6 files changed, 180 insertions(+), 25 deletions(-)

diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index f553523857..3989f6e54a 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -104,6 +104,7 @@ int			wal_level = WAL_LEVEL_MINIMAL;
 int			CommitDelay = 0;	/* precommit delay in microseconds */
 int			CommitSiblings = 5; /* # concurrent xacts needed to sleep */
 int			wal_retrieve_retry_interval = 5000;
+int			max_slot_wal_keep_size_mb = -1;
 
 #ifdef WAL_DEBUG
 bool		XLOG_DEBUG = false;
@@ -872,6 +873,7 @@ static void checkTimeLineSwitch(XLogRecPtr lsn, TimeLineID newTLI,
 static void LocalSetXLogInsertAllowed(void);
 static void CreateEndOfRecoveryRecord(void);
 static void CheckPointGuts(XLogRecPtr checkPointRedo, int flags);
+static XLogSegNo GetOldestKeepSegment(XLogRecPtr currpos, XLogRecPtr minSlotPtr);
 static void KeepLogSeg(XLogRecPtr recptr, XLogSegNo *logSegNo);
 static XLogRecPtr XLogGetReplicationSlotMinimumLSN(void);
 
@@ -9288,6 +9290,54 @@ CreateRestartPoint(int flags)
 	return true;
 }
 
+/*
+ * Returns minimum segment number that the next checkpoint must leave
+ * considering wal_keep_segments, replication slots and
+ * max_slot_wal_keep_size.
+ *
+ * currLSN is the current insert location.
+ * minSlotLSN is the minimum restart_lsn of all active slots.
+ */
+static XLogSegNo
+GetOldestKeepSegment(XLogRecPtr currLSN, XLogRecPtr minSlotLSN)
+{
+	XLogSegNo	currSeg;
+	XLogSegNo	minSlotSeg;
+	uint64		keepSegs = 0;	/* # of segments actually kept */
+
+	XLByteToSeg(currLSN, currSeg, wal_segment_size);
+	XLByteToSeg(minSlotLSN, minSlotSeg, wal_segment_size);
+
+	/*
+	 * Calculate how many segments are kept by slots first. The second
+	 * term of the condition is just a sanity check.
+	 */
+	if (minSlotLSN != InvalidXLogRecPtr && minSlotSeg <= currSeg)
+		keepSegs = currSeg - minSlotSeg;
+
+	/* Cap keepSegs by max_slot_wal_keep_size */
+	if (max_slot_wal_keep_size_mb >= 0)
+	{
+		uint64 limitSegs;
+
+		limitSegs = ConvertToXSegs(max_slot_wal_keep_size_mb, wal_segment_size);
+
+		/* Reduce it if slots already reserves too many. */
+		if (limitSegs < keepSegs)
+			keepSegs = limitSegs;
+	}
+
+	/* but, keep at least wal_keep_segments segments if any */
+	if (wal_keep_segments > 0 && keepSegs < wal_keep_segments)
+		keepSegs = wal_keep_segments;
+
+	/* avoid underflow, don't go below 1 */
+	if (currSeg <= keepSegs)
+		return 1;
+
+	return currSeg - keepSegs;
+}
+
 /*
  * Retreat *logSegNo to the last segment that we need to retain because of
  * either wal_keep_segments or replication slots.
@@ -9299,38 +9349,66 @@ CreateRestartPoint(int flags)
 static void
 KeepLogSeg(XLogRecPtr recptr, XLogSegNo *logSegNo)
 {
-	XLogSegNo	segno;
-	XLogRecPtr	keep;
+	XLogRecPtr	slotminptr = InvalidXLogRecPtr;
+	XLogSegNo	minSegNo;
+	XLogSegNo	slotSegNo;
 
-	XLByteToSeg(recptr, segno, wal_segment_size);
-	keep = XLogGetReplicationSlotMinimumLSN();
+	if (max_replication_slots > 0)
+		slotminptr = XLogGetReplicationSlotMinimumLSN();
 
-	/* compute limit for wal_keep_segments first */
-	if (wal_keep_segments > 0)
+	/*
+	 * We should keep certain number of WAL segments after this checkpoint.
+	 */
+	minSegNo = GetOldestKeepSegment(recptr, slotminptr);
+
+	/*
+	 * Warn the checkpoint is going to flush the segments required by
+	 * replication slots.
+	 */
+	if (!XLogRecPtrIsInvalid(slotminptr))
 	{
-		/* avoid underflow, don't go below 1 */
-		if (segno <= wal_keep_segments)
-			segno = 1;
+		static XLogSegNo prev_lost_segs = 0;	/* avoid duplicate messages */
+
+		XLByteToSeg(slotminptr, slotSegNo, wal_segment_size);
+
+		if (slotSegNo < minSegNo)
+		{
+			XLogSegNo lost_segs = minSegNo - slotSegNo;
+			if (prev_lost_segs != lost_segs)
+			{
+				/* We have lost a new segment, warn it.*/
+				XLogRecPtr minlsn;
+				char *slot_names;
+				int nslots;
+
+				XLogSegNoOffsetToRecPtr(minSegNo, 0, wal_segment_size, minlsn);
+				slot_names =
+					ReplicationSlotsEnumerateBehinds(minlsn, ", ", &nslots);
+
+				/*
+				 * Some of the affected slots could have just been removed. We
+				 * don't need show anything here if no affected slots are
+				 * remaining.
+				 */
+				if (slot_names)
+				{
+					ereport(WARNING,
+							(errmsg ("some replication slots have lost required WAL segments"),
+							 errdetail_plural(
+								 "Slot %s lost %ld segment(s).",
+								 "Slots %s lost at most %ld segment(s).",
+								 nslots, slot_names, lost_segs)));
+				}
+			}
+			prev_lost_segs = lost_segs;
+		}
 		else
-			segno = segno - wal_keep_segments;
-	}
-
-	/* then check whether slots limit removal further */
-	if (max_replication_slots > 0 && keep != InvalidXLogRecPtr)
-	{
-		XLogSegNo	slotSegNo;
-
-		XLByteToSeg(keep, slotSegNo, wal_segment_size);
-
-		if (slotSegNo <= 0)
-			segno = 1;
-		else if (slotSegNo < segno)
-			segno = slotSegNo;
+			prev_lost_segs = 0;
 	}
 
 	/* don't delete WAL segments newer than the calculated segment */
-	if (segno < *logSegNo)
-		*logSegNo = segno;
+	if (minSegNo < *logSegNo)
+		*logSegNo = minSegNo;
 }
 
 /*
diff --git a/src/backend/replication/slot.c b/src/backend/replication/slot.c
index 62342a69cb..5bdf1e90fb 100644
--- a/src/backend/replication/slot.c
+++ b/src/backend/replication/slot.c
@@ -1064,6 +1064,68 @@ ReplicationSlotReserveWal(void)
 	}
 }
 
+/*
+ * Returns names of inactive replication slots that their restart_lsn are
+ * behind specified LSN for the purpose of error message, in palloc'ed
+ * character array stuffed with slot names delimited by the given
+ * separator. Returns NULL if no slot matches. If pnslots is given, the number
+ * of the returned slots is returned there.
+ */
+char *
+ReplicationSlotsEnumerateBehinds(XLogRecPtr target, char *separator, int *pnslots)
+{
+	static StringInfoData retstr;
+	static bool retstr_initialized = false;
+	bool insert_separator = false;
+	int i;
+	int nslots = 0;
+
+	Assert (separator);
+	if (max_replication_slots <= 0)
+		return NULL;
+
+	if (!retstr_initialized)
+	{
+		initStringInfo(&retstr);
+		retstr_initialized = true;
+	}
+	else
+		resetStringInfo(&retstr);
+
+	/* construct name list */
+	LWLockAcquire(ReplicationSlotControlLock, LW_SHARED);
+	for (i = 0 ; i < max_replication_slots ; i++)
+	{
+		ReplicationSlot *s = &ReplicationSlotCtl->replication_slots[i];
+
+		/*
+		 * We are collecting slots that are definitely behind the given target
+		 * LSN. Active slots are exluded since they can catch up later.
+		 */
+		if (s->in_use && s->active_pid == 0 && s->data.restart_lsn < target)
+		{
+			if (insert_separator)
+				appendStringInfoString(&retstr, separator);
+
+			/*
+			 * Slot names consist only with lower-case letters. We don't
+			 * bother quoting.
+			 */
+			appendStringInfoString(&retstr, NameStr(s->data.name));
+			insert_separator = true;
+			nslots++;
+		}
+	}
+	LWLockRelease(ReplicationSlotControlLock);
+
+	/* return the number of slots in the list if requested */
+	if (pnslots)
+		*pnslots = nslots;
+
+	/* return NULL instead of an empty string */
+	return retstr.data[0] ? retstr.data : NULL;
+}
+
 /*
  * Flush all replication slots to disk.
  *
diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c
index fc463601ff..f8e796b6c1 100644
--- a/src/backend/utils/misc/guc.c
+++ b/src/backend/utils/misc/guc.c
@@ -2654,6 +2654,18 @@ static struct config_int ConfigureNamesInt[] =
 		NULL, NULL, NULL
 	},
 
+	{
+		{"max_slot_wal_keep_size", PGC_SIGHUP, REPLICATION_SENDING,
+			gettext_noop("Sets the maximum size of extra WALs kept by replication slots."),
+		 NULL,
+		 GUC_UNIT_MB
+		},
+		&max_slot_wal_keep_size_mb,
+		-1, -1,
+		MAX_KILOBYTES, /* XXX: This is in megabytes, like max/min_wal_size */
+		NULL, NULL, NULL
+	},
+
 	{
 		{"wal_sender_timeout", PGC_USERSET, REPLICATION_SENDING,
 			gettext_noop("Sets the maximum time to wait for WAL replication."),
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index cfad86c02a..aadbc76d85 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -286,6 +286,7 @@
 #max_wal_senders = 10		# max number of walsender processes
 				# (change requires restart)
 #wal_keep_segments = 0		# in logfile segments; 0 disables
+#max_slot_wal_keep_size = -1	# measured in bytes; -1 disables
 #wal_sender_timeout = 60s	# in milliseconds; 0 disables
 
 #max_replication_slots = 10	# max number of replication slots
diff --git a/src/include/access/xlog.h b/src/include/access/xlog.h
index d519252aad..b355452072 100644
--- a/src/include/access/xlog.h
+++ b/src/include/access/xlog.h
@@ -108,6 +108,7 @@ extern int	wal_segment_size;
 extern int	min_wal_size_mb;
 extern int	max_wal_size_mb;
 extern int	wal_keep_segments;
+extern int	max_slot_wal_keep_size_mb;
 extern int	XLOGbuffers;
 extern int	XLogArchiveTimeout;
 extern int	wal_retrieve_retry_interval;
diff --git a/src/include/replication/slot.h b/src/include/replication/slot.h
index 8fbddea78f..e0fee0663c 100644
--- a/src/include/replication/slot.h
+++ b/src/include/replication/slot.h
@@ -199,6 +199,7 @@ extern void ReplicationSlotsComputeRequiredLSN(void);
 extern XLogRecPtr ReplicationSlotsComputeLogicalRestartLSN(void);
 extern bool ReplicationSlotsCountDBSlots(Oid dboid, int *nslots, int *nactive);
 extern void ReplicationSlotsDropDBSlots(Oid dboid);
+extern char *ReplicationSlotsEnumerateBehinds(XLogRecPtr target, char *separator, int *pnslots);
 
 extern void StartupReplicationSlots(void);
 extern void CheckPointReplicationSlots(void);
-- 
2.16.3


----Next_Part(Wed_Jul_31_16_56_16_2019_668)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v16-0002-Add-monitoring-aid-for-max_slot_wal_keep_size.patch"



^ permalink  raw  reply  [nested|flat] 125+ messages in thread

* [PATCH 1/6] Add WAL relief vent for replication slots
@ 2017-12-21 12:20 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 125+ messages in thread

From: Kyotaro Horiguchi @ 2017-12-21 12:20 UTC (permalink / raw)

Replication slot is useful to maintain replication connection in the
configurations where replication is so delayed that connection is
broken. On the other hand so many WAL files can fill up disk that the
master downs by a long delay. This feature, which is activated by a
GUC "max_slot_wal_keep_size", protects master servers from suffering
disk full by limiting the number of WAL files reserved by replication
slots.
---
 src/backend/access/transam/xlog.c             | 128 +++++++++++++++++++++-----
 src/backend/replication/slot.c                |  58 ++++++++++++
 src/backend/utils/misc/guc.c                  |  12 +++
 src/backend/utils/misc/postgresql.conf.sample |   1 +
 src/include/access/xlog.h                     |   1 +
 src/include/replication/slot.h                |   1 +
 6 files changed, 176 insertions(+), 25 deletions(-)

diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index f553523857..fcb076100f 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -104,6 +104,7 @@ int			wal_level = WAL_LEVEL_MINIMAL;
 int			CommitDelay = 0;	/* precommit delay in microseconds */
 int			CommitSiblings = 5; /* # concurrent xacts needed to sleep */
 int			wal_retrieve_retry_interval = 5000;
+int			max_slot_wal_keep_size_mb = -1;
 
 #ifdef WAL_DEBUG
 bool		XLOG_DEBUG = false;
@@ -872,6 +873,7 @@ static void checkTimeLineSwitch(XLogRecPtr lsn, TimeLineID newTLI,
 static void LocalSetXLogInsertAllowed(void);
 static void CreateEndOfRecoveryRecord(void);
 static void CheckPointGuts(XLogRecPtr checkPointRedo, int flags);
+static XLogSegNo GetOldestKeepSegment(XLogRecPtr currpos, XLogRecPtr minSlotPtr);
 static void KeepLogSeg(XLogRecPtr recptr, XLogSegNo *logSegNo);
 static XLogRecPtr XLogGetReplicationSlotMinimumLSN(void);
 
@@ -9288,6 +9290,54 @@ CreateRestartPoint(int flags)
 	return true;
 }
 
+/*
+ * Returns minimum segment number that the next checkpoint must leave
+ * considering wal_keep_segments, replication slots and
+ * max_slot_wal_keep_size.
+ *
+ * currLSN is the current insert location.
+ * minSlotLSN is the minimum restart_lsn of all active slots.
+ */
+static XLogSegNo
+GetOldestKeepSegment(XLogRecPtr currLSN, XLogRecPtr minSlotLSN)
+{
+	XLogSegNo	currSeg;
+	XLogSegNo	minSlotSeg;
+	uint64		keepSegs = 0;	/* # of segments actually kept */
+
+	XLByteToSeg(currLSN, currSeg, wal_segment_size);
+	XLByteToSeg(minSlotLSN, minSlotSeg, wal_segment_size);
+
+	/*
+	 * Calculate how many segments are kept by slots first. The second
+	 * term of the condition is just a sanity check.
+	 */
+	if (minSlotLSN != InvalidXLogRecPtr && minSlotSeg <= currSeg)
+		keepSegs = currSeg - minSlotSeg;
+
+	/* Cap keepSegs by max_slot_wal_keep_size */
+	if (max_slot_wal_keep_size_mb >= 0)
+	{
+		uint64 limitSegs;
+
+		limitSegs = ConvertToXSegs(max_slot_wal_keep_size_mb, wal_segment_size);
+
+		/* Reduce it if slots already reserves too many. */
+		if (limitSegs < keepSegs)
+			keepSegs = limitSegs;
+	}
+
+	/* but, keep at least wal_keep_segments segments if any */
+	if (wal_keep_segments > 0 && keepSegs < wal_keep_segments)
+		keepSegs = wal_keep_segments;
+
+	/* avoid underflow, don't go below 1 */
+	if (currSeg <= keepSegs)
+		return 1;
+
+	return currSeg - keepSegs;
+}
+
 /*
  * Retreat *logSegNo to the last segment that we need to retain because of
  * either wal_keep_segments or replication slots.
@@ -9299,38 +9349,66 @@ CreateRestartPoint(int flags)
 static void
 KeepLogSeg(XLogRecPtr recptr, XLogSegNo *logSegNo)
 {
-	XLogSegNo	segno;
-	XLogRecPtr	keep;
+	XLogRecPtr	slotminptr = InvalidXLogRecPtr;
+	XLogSegNo	minSegNo;
+	XLogSegNo	slotSegNo;
 
-	XLByteToSeg(recptr, segno, wal_segment_size);
-	keep = XLogGetReplicationSlotMinimumLSN();
+	if (max_replication_slots > 0)
+		slotminptr = XLogGetReplicationSlotMinimumLSN();
 
-	/* compute limit for wal_keep_segments first */
-	if (wal_keep_segments > 0)
+	/*
+	 * We should keep certain number of WAL segments after this checkpoint.
+	 */
+	minSegNo = GetOldestKeepSegment(recptr, slotminptr);
+
+	/*
+	 * Warn the checkpoint is going to flush the segments required by
+	 * replication slots.
+	 */
+	if (!XLogRecPtrIsInvalid(slotminptr))
 	{
-		/* avoid underflow, don't go below 1 */
-		if (segno <= wal_keep_segments)
-			segno = 1;
+		static XLogSegNo prev_lost_segs = 0;	/* avoid duplicate messages */
+
+		XLByteToSeg(slotminptr, slotSegNo, wal_segment_size);
+
+		if (slotSegNo < minSegNo)
+		{
+			XLogSegNo lost_segs = minSegNo - slotSegNo;
+			if (prev_lost_segs != lost_segs)
+			{
+				/* We have lost a new segment, warn it.*/
+				XLogRecPtr minlsn;
+				char *slot_names;
+				int nslots;
+
+				XLogSegNoOffsetToRecPtr(minSegNo, 0, wal_segment_size, minlsn);
+				slot_names =
+					ReplicationSlotsEnumerateBehinds(minlsn, ", ", &nslots);
+
+				/*
+				 * Some of the affected slots could have just been removed.
+				 * We don't need show anything here if no affected slot
+				 * remains.
+				 */
+				if (slot_names)
+				{
+					ereport(WARNING,
+							(errmsg ("some replication slots have lost required WAL segments"),
+							 errdetail_plural(
+								 "Slot %s lost %ld segment(s).",
+								 "Slots %s lost at most %ld segment(s).",
+								 nslots, slot_names, lost_segs)));
+				}
+			}
+			prev_lost_segs = lost_segs;
+		}
 		else
-			segno = segno - wal_keep_segments;
-	}
-
-	/* then check whether slots limit removal further */
-	if (max_replication_slots > 0 && keep != InvalidXLogRecPtr)
-	{
-		XLogSegNo	slotSegNo;
-
-		XLByteToSeg(keep, slotSegNo, wal_segment_size);
-
-		if (slotSegNo <= 0)
-			segno = 1;
-		else if (slotSegNo < segno)
-			segno = slotSegNo;
+			prev_lost_segs = 0;
 	}
 
 	/* don't delete WAL segments newer than the calculated segment */
-	if (segno < *logSegNo)
-		*logSegNo = segno;
+	if (minSegNo < *logSegNo)
+		*logSegNo = minSegNo;
 }
 
 /*
diff --git a/src/backend/replication/slot.c b/src/backend/replication/slot.c
index 62342a69cb..24b8d42eab 100644
--- a/src/backend/replication/slot.c
+++ b/src/backend/replication/slot.c
@@ -1064,6 +1064,64 @@ ReplicationSlotReserveWal(void)
 	}
 }
 
+/*
+ * Returns names of replication slots that their restart_lsn are behind
+ * specified LSN, in palloc'ed character array stuffed with slot names
+ * delimited by the given separator.  Returns NULL if no slot matches. If
+ * pnslots is given, the number of the returned slots is returned there.
+ */
+char *
+ReplicationSlotsEnumerateBehinds(XLogRecPtr target, char *separator, int *pnslots)
+{
+	static StringInfoData retstr;
+	static bool retstr_initialized = false;
+	bool insert_separator = false;
+	int i;
+	int nslots = 0;
+
+	Assert (separator);
+	if (max_replication_slots <= 0)
+		return NULL;
+
+	if (!retstr_initialized)
+	{
+		initStringInfo(&retstr);
+		retstr_initialized = true;
+	}
+	else
+		resetStringInfo(&retstr);
+
+	/* construct name list */
+	LWLockAcquire(ReplicationSlotControlLock, LW_SHARED);
+	for (i = 0 ; i < max_replication_slots ; i++)
+	{
+		ReplicationSlot *s = &ReplicationSlotCtl->replication_slots[i];
+
+		/* Exclude active walsenders */
+		if (s->in_use && s->active_pid == 0 && s->data.restart_lsn < target)
+		{
+			if (insert_separator)
+				appendStringInfoString(&retstr, separator);
+
+			/*
+			 * Slot names consist only with lower-case letters. We don't
+			 * bother quoting.
+			 */
+			appendStringInfoString(&retstr, NameStr(s->data.name));
+			insert_separator = true;
+			nslots++;
+		}
+	}
+	LWLockRelease(ReplicationSlotControlLock);
+
+	/* return the number of slots in the list if requested */
+	if (pnslots)
+		*pnslots = nslots;
+
+	/* return NULL instead of an empty string */
+	return retstr.data[0] ? retstr.data : NULL;
+}
+
 /*
  * Flush all replication slots to disk.
  *
diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c
index fc463601ff..f8e796b6c1 100644
--- a/src/backend/utils/misc/guc.c
+++ b/src/backend/utils/misc/guc.c
@@ -2654,6 +2654,18 @@ static struct config_int ConfigureNamesInt[] =
 		NULL, NULL, NULL
 	},
 
+	{
+		{"max_slot_wal_keep_size", PGC_SIGHUP, REPLICATION_SENDING,
+			gettext_noop("Sets the maximum size of extra WALs kept by replication slots."),
+		 NULL,
+		 GUC_UNIT_MB
+		},
+		&max_slot_wal_keep_size_mb,
+		-1, -1,
+		MAX_KILOBYTES, /* XXX: This is in megabytes, like max/min_wal_size */
+		NULL, NULL, NULL
+	},
+
 	{
 		{"wal_sender_timeout", PGC_USERSET, REPLICATION_SENDING,
 			gettext_noop("Sets the maximum time to wait for WAL replication."),
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index cfad86c02a..aadbc76d85 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -286,6 +286,7 @@
 #max_wal_senders = 10		# max number of walsender processes
 				# (change requires restart)
 #wal_keep_segments = 0		# in logfile segments; 0 disables
+#max_slot_wal_keep_size = -1	# measured in bytes; -1 disables
 #wal_sender_timeout = 60s	# in milliseconds; 0 disables
 
 #max_replication_slots = 10	# max number of replication slots
diff --git a/src/include/access/xlog.h b/src/include/access/xlog.h
index d519252aad..b355452072 100644
--- a/src/include/access/xlog.h
+++ b/src/include/access/xlog.h
@@ -108,6 +108,7 @@ extern int	wal_segment_size;
 extern int	min_wal_size_mb;
 extern int	max_wal_size_mb;
 extern int	wal_keep_segments;
+extern int	max_slot_wal_keep_size_mb;
 extern int	XLOGbuffers;
 extern int	XLogArchiveTimeout;
 extern int	wal_retrieve_retry_interval;
diff --git a/src/include/replication/slot.h b/src/include/replication/slot.h
index 8fbddea78f..e0fee0663c 100644
--- a/src/include/replication/slot.h
+++ b/src/include/replication/slot.h
@@ -199,6 +199,7 @@ extern void ReplicationSlotsComputeRequiredLSN(void);
 extern XLogRecPtr ReplicationSlotsComputeLogicalRestartLSN(void);
 extern bool ReplicationSlotsCountDBSlots(Oid dboid, int *nslots, int *nactive);
 extern void ReplicationSlotsDropDBSlots(Oid dboid);
+extern char *ReplicationSlotsEnumerateBehinds(XLogRecPtr target, char *separator, int *pnslots);
 
 extern void StartupReplicationSlots(void);
 extern void CheckPointReplicationSlots(void);
-- 
2.16.3


----Next_Part(Tue_Jul_30_21_30_45_2019_680)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v15-0002-Add-monitoring-aid-for-max_slot_wal_keep_size.patch"



^ permalink  raw  reply  [nested|flat] 125+ messages in thread

* [PATCH v21 1/2] Add WAL relief vent for replication slots
@ 2018-12-19 03:43 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 125+ messages in thread

From: Kyotaro Horiguchi @ 2018-12-19 03:43 UTC (permalink / raw)

Replication slot is useful to maintain replication connection in the
configurations where replication is so delayed that connection is
broken. On the other hand so many WAL files can fill up disk that the
master downs by a long delay. This feature, which is activated by a
GUC "max_slot_wal_keep_size", protects master servers from suffering
disk full by limiting the number of WAL files reserved by replication
slots.
---
 doc/src/sgml/catalogs.sgml                    |  48 +++
 doc/src/sgml/config.sgml                      |  23 ++
 doc/src/sgml/high-availability.sgml           |   8 +-
 src/backend/access/transam/xlog.c             | 322 ++++++++++++++++--
 src/backend/catalog/system_views.sql          |   4 +-
 src/backend/replication/slot.c                |   1 +
 src/backend/replication/slotfuncs.c           |  39 ++-
 src/backend/utils/misc/guc.c                  |  13 +
 src/backend/utils/misc/postgresql.conf.sample |   1 +
 src/include/access/xlog.h                     |  19 ++
 src/include/catalog/pg_proc.dat               |   6 +-
 src/test/recovery/t/018_replslot_limit.pl     | 203 +++++++++++
 src/test/regress/expected/rules.out           |   6 +-
 13 files changed, 657 insertions(+), 36 deletions(-)
 create mode 100644 src/test/recovery/t/018_replslot_limit.pl

diff --git a/doc/src/sgml/catalogs.sgml b/doc/src/sgml/catalogs.sgml
index 64614b569c..01a7802ed4 100644
--- a/doc/src/sgml/catalogs.sgml
+++ b/doc/src/sgml/catalogs.sgml
@@ -9907,6 +9907,54 @@ SELECT * FROM pg_locks pl LEFT JOIN pg_prepared_xacts ppx
       </entry>
      </row>
 
+     <row>
+      <entry><structfield>wal_status</structfield></entry>
+      <entry><type>text</type></entry>
+      <entry></entry>
+
+      <entry>Availability of WAL files claimed by this slot.
+      Valid values are:
+       <simplelist>
+        <member>
+         <literal>normal</literal> means that the claimed files
+         are within <varname>max_wal_size</varname>
+        </member>
+        <member>
+         <literal>keeping</literal> means that <varname>max_wal_size</varname>
+         is exceeded but still held by replication slots or
+         <varname>wal_keep_segments</varname>
+        </member>
+        <member>
+         <literal>losing</literal> means that some of the files are on the verge
+         of deletion, but can still be accessed by a session that's currently
+         reading it
+        </member>
+        <member>
+         <literal>lost</literal> means that some of them are definitely lost
+         and the session using this slot cannot continue replication.
+         This state also implies that the session using this slot has been
+         stopped.
+        </member>
+       </simplelist>
+      The last two states are seen only when
+      <xref linkend="guc-max-slot-wal-keep-size"/> is
+      non-negative. If <structfield>restart_lsn</structfield> is NULL, this
+      field is null.
+      </entry>
+     </row>
+
+     <row>
+      <entry><structfield>remain</structfield></entry>
+      <entry><type>bigint</type></entry>
+      <entry></entry>
+      <entry>The amount in bytes of WAL that can be written before this slot
+        loses required WAL files.
+        If <structfield>restart_lsn</structfield> is null or
+        <structfield>wal_status</structfield> is <literal>losing</literal>
+        or <literal>lost</literal>, this field is null.
+      </entry>
+     </row>
+
     </tbody>
    </tgroup>
   </table>
diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index 2de21903a1..dc99c6868a 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -3758,6 +3758,29 @@ restore_command = 'copy "C:\\server\\archivedir\\%f" "%p"'  # Windows
       </listitem>
      </varlistentry>
 
+      <varlistentry id="guc-max-slot-wal-keep-size" xreflabel="max_slot_wal_keep_size">
+       <term><varname>max_slot_wal_keep_size</varname> (<type>integer</type>)
+       <indexterm>
+        <primary><varname>max_slot_wal_keep_size</varname> configuration parameter</primary>
+       </indexterm>
+       </term>
+       <listitem>
+       <para>
+        Specify the maximum size of WAL files
+        that <link linkend="streaming-replication-slots">replication
+        slots</link> are allowed to retain in the <filename>pg_wal</filename>
+        directory at checkpoint time.
+        If <varname>max_slot_wal_keep_size</varname> is -1 (the default),
+        replication slots retain unlimited amount of WAL files.  If
+        restart_lsn of a replication slot gets behind more than that megabytes
+        from the current LSN, the standby using the slot may no longer be able
+        to continue replication due to removal of required WAL files. You
+        can see the WAL availability of replication slots
+        in <link linkend="view-pg-replication-slots">pg_replication_slots</link>.
+       </para>
+       </listitem>
+      </varlistentry>
+
      <varlistentry id="guc-wal-sender-timeout" xreflabel="wal_sender_timeout">
       <term><varname>wal_sender_timeout</varname> (<type>integer</type>)
       <indexterm>
diff --git a/doc/src/sgml/high-availability.sgml b/doc/src/sgml/high-availability.sgml
index b5d32bb720..624e5f94ad 100644
--- a/doc/src/sgml/high-availability.sgml
+++ b/doc/src/sgml/high-availability.sgml
@@ -925,9 +925,11 @@ primary_conninfo = 'host=192.168.1.50 port=5432 user=foo password=foopass'
     <xref linkend="guc-archive-command"/>.
     However, these methods often result in retaining more WAL segments than
     required, whereas replication slots retain only the number of segments
-    known to be needed.  An advantage of these methods is that they bound
-    the space requirement for <literal>pg_wal</literal>; there is currently no way
-    to do this using replication slots.
+    known to be needed.  On the other hand, replication slots can retain so
+    many WAL segments that they fill up the space allocated
+    for <literal>pg_wal</literal>;
+    <xref linkend="guc-max-slot-wal-keep-size"/> limits the size of WAL files
+    retained by replication slots.
    </para>
    <para>
     Similarly, <xref linkend="guc-hot-standby-feedback"/>
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 977d448f50..db7494e741 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -106,6 +106,7 @@ int			wal_level = WAL_LEVEL_MINIMAL;
 int			CommitDelay = 0;	/* precommit delay in microseconds */
 int			CommitSiblings = 5; /* # concurrent xacts needed to sleep */
 int			wal_retrieve_retry_interval = 5000;
+int			max_slot_wal_keep_size_mb = -1;
 
 #ifdef WAL_DEBUG
 bool		XLOG_DEBUG = false;
@@ -759,7 +760,7 @@ static ControlFileData *ControlFile = NULL;
  */
 #define UsableBytesInPage (XLOG_BLCKSZ - SizeOfXLogShortPHD)
 
-/* Convert min_wal_size_mb and max_wal_size_mb to equivalent segment count */
+/* Convert values of GUCs measured in megabytes to equiv. segment count */
 #define ConvertToXSegs(x, segsize)	\
 	(x / ((segsize) / (1024 * 1024)))
 
@@ -896,6 +897,7 @@ static void checkTimeLineSwitch(XLogRecPtr lsn, TimeLineID newTLI,
 static void LocalSetXLogInsertAllowed(void);
 static void CreateEndOfRecoveryRecord(void);
 static void CheckPointGuts(XLogRecPtr checkPointRedo, int flags);
+static XLogSegNo GetOldestKeepSegment(XLogRecPtr currpos, XLogRecPtr minSlotPtr);
 static void KeepLogSeg(XLogRecPtr recptr, XLogSegNo *logSegNo);
 static XLogRecPtr XLogGetReplicationSlotMinimumLSN(void);
 
@@ -3930,8 +3932,21 @@ XLogGetLastRemovedSegno(void)
 }
 
 /*
- * Update the last removed segno pointer in shared memory, to reflect
- * that the given XLOG file has been removed.
+ * Return the oldest extant WAL segment file.
+ *
+ * When XLogGetLastRemovedSegno returns a non-zero value, then the oldest
+ * valid segment is one further from that; and if it returns zero, then
+ * it is the segment numbered on.
+ */
+XLogSegNo
+FindOldestXLogFileSegNo(void)
+{
+	return XLogGetLastRemovedSegno() + 1;
+}
+
+/*
+ * Update the last removed segno pointer in shared memory, to reflect that the
+ * given XLOG file has been removed.
  */
 static void
 UpdateLastRemovedPtr(char *filename)
@@ -9451,6 +9466,197 @@ CreateRestartPoint(int flags)
 	return true;
 }
 
+/*
+ * Detect availability of the record at given targetLSN.
+ *
+ * targetLSN is restart_lsn of a slot.
+ * walsender_pid is the slot's walsender PID.
+ *
+ * Returns one of the following enum values.
+ *
+ * WALAVAIL_NORMAL means targetLSN is available because it is in the range of
+ * max_wal_size.  If max_slot_wal_keep_size is smaller than max_wal_size, this
+ * state is not returned.
+ *
+ * WALAVAIL_PRESERVED means it is still available by preserving extra segments
+ * beyond max_wal_size.
+ *
+ * WALAVAIL_BEING_REMOVED means it is being removed or already removed but the
+ * replication stream on the given slot is live yet. The state may transit to
+ * WALAVAIL_PRESERVED or WALAVAIL_NORMAL state if the walsender advances
+ * restart_lsn.
+ *
+ * WALAVAIL_REMOVED means it is definitely lost. The replication stream on the
+ * slot cannot continue.
+ *
+ * returns WALAVAIL_NULL if restart_lsn is invalid.
+ */
+WalAvailability
+GetWalAvailability(XLogRecPtr restart_lsn, pid_t walsender_pid)
+{
+	XLogRecPtr currpos;
+	XLogRecPtr slotPtr;
+	XLogSegNo currSeg;		/* segid of currpos */
+	XLogSegNo restartSeg;	/* segid of restart_lsn */
+	XLogSegNo oldestSeg;	/* actual oldest segid */
+	XLogSegNo oldestSegMaxWalSize;	/* oldest segid kept by max_wal_size */
+	XLogSegNo oldestSlotSeg;/* oldest segid kept by slot */
+	uint64	  keepSegs;
+
+	/* the case where the slot has never been activated */
+	if (XLogRecPtrIsInvalid(restart_lsn))
+		return WALAVAIL_INVALID_LSN;
+
+	currpos = GetXLogWriteRecPtr();
+
+	/* calculate oldest segment currently needed by slots */
+	XLByteToSeg(restart_lsn, restartSeg, wal_segment_size);
+	slotPtr = XLogGetReplicationSlotMinimumLSN();
+	oldestSlotSeg = GetOldestKeepSegment(currpos, slotPtr);
+
+	/* find the oldest extant segment file */
+	oldestSeg = FindOldestXLogFileSegNo();
+
+	/* calculate oldest segment by max_wal_size */
+	XLByteToSeg(currpos, currSeg, wal_segment_size);
+	keepSegs = ConvertToXSegs(max_wal_size_mb, wal_segment_size) + 1;
+
+	if (currSeg > keepSegs)
+		oldestSegMaxWalSize = currSeg - keepSegs;
+	else
+		oldestSegMaxWalSize = 1;
+
+	/*
+	 * If max_slot_wal_keep_size has changed after the last call, the segment
+	 * that would been kept by the current setting might have been lost by the
+	 * previous setting. No point in showing normal or keeping status values if
+	 * the restartSeg is known to be lost.
+	 */
+	if (restartSeg >= oldestSeg)
+	{
+		/*
+		 * show "normal" when restartSeg is within max_wal_size. If
+		 * max_slot_wal_keep_size is smaller than max_wal_size, there's no
+		 * point in showing the status.
+		 */
+		if ((max_slot_wal_keep_size_mb <= 0 ||
+			 max_slot_wal_keep_size_mb >= max_wal_size_mb) &&
+			oldestSegMaxWalSize <= restartSeg)
+			return WALAVAIL_NORMAL;
+
+		/* being retained by slots */
+		if (oldestSlotSeg <= restartSeg)
+			return WALAVAIL_PRESERVED;
+	}
+	
+	/*
+	 * The segment is already lost or being lost. If the oldest segment is just
+	 * after the restartSeg, running walsender may be reading the just removed
+	 * segment. The walsender may safely move to the oldest existing segment in
+	 * that case.
+	 */
+	if (oldestSeg == restartSeg + 1 && walsender_pid != 0)
+		return WALAVAIL_BEING_REMOVED;
+
+	/* definitely lost. the walsender can no longer restart */
+	return WALAVAIL_REMOVED;
+}
+
+/*
+ * Returns minimum segment number that the next checkpoint must leave
+ * considering wal_keep_segments, replication slots and
+ * max_slot_wal_keep_size.
+ *
+ * currLSN is the current insert location.
+ * minSlotLSN is the minimum restart_lsn of all active slots.
+ */
+static XLogSegNo
+GetOldestKeepSegment(XLogRecPtr currLSN, XLogRecPtr minSlotLSN)
+{
+	XLogSegNo	currSeg;
+	XLogSegNo	minSlotSeg;
+	uint64		keepSegs = 0;	/* # of segments actually kept */
+
+	XLByteToSeg(currLSN, currSeg, wal_segment_size);
+	XLByteToSeg(minSlotLSN, minSlotSeg, wal_segment_size);
+
+	/*
+	 * Calculate how many segments are kept by slots first. The second
+	 * term of the condition is just a sanity check.
+	 */
+	if (minSlotLSN != InvalidXLogRecPtr && minSlotSeg <= currSeg)
+		keepSegs = currSeg - minSlotSeg;
+
+	/* Cap keepSegs by max_slot_wal_keep_size */
+	if (max_slot_wal_keep_size_mb >= 0)
+	{
+		uint64 limitSegs;
+
+		limitSegs = ConvertToXSegs(max_slot_wal_keep_size_mb, wal_segment_size);
+
+		/* Reduce it if slots already reserves too many. */
+		if (limitSegs < keepSegs)
+			keepSegs = limitSegs;
+	}
+
+	/* but, keep at least wal_keep_segments segments if any */
+	if (wal_keep_segments > 0 && keepSegs < wal_keep_segments)
+		keepSegs = wal_keep_segments;
+
+	/* avoid underflow, don't go below 1 */
+	if (currSeg <= keepSegs)
+		return 1;
+
+	return currSeg - keepSegs;
+}
+
+/*
+ * Calculate remaining bytes until WAL segment for targetLSN will be removed.
+ */
+int64
+DistanceToWalRemoval(XLogRecPtr currLSN, XLogRecPtr targetLSN)
+{
+	XLogSegNo	currSeg;
+	uint64		limitSegs = 0;
+	int64 		restbytes;
+	uint64		fragbytes;
+	XLogSegNo	targetSeg;
+
+	XLByteToSeg(currLSN, currSeg, wal_segment_size);
+
+	/* Calculate how far back WAL segments are preserved */
+	if (max_slot_wal_keep_size_mb >= 0)
+		limitSegs = ConvertToXSegs(max_slot_wal_keep_size_mb, wal_segment_size);
+
+	if (wal_keep_segments > 0 && limitSegs < wal_keep_segments)
+		limitSegs = wal_keep_segments;
+
+	XLByteToSeg(targetLSN, targetSeg, wal_segment_size);
+
+	/* avoid underflow */
+	if (targetSeg + limitSegs < currSeg)
+		return 0;
+
+	/*
+	 * This slot still has all required segments. Calculate how
+	 * many LSN bytes the slot has until it loses targetLSN.
+	 */
+	fragbytes = wal_segment_size - (currLSN % wal_segment_size);
+	XLogSegNoOffsetToRecPtr(targetSeg + limitSegs - currSeg,
+							fragbytes, wal_segment_size,
+							restbytes);
+
+	/*
+	 * Not realistic, but make sure that it is not out of the
+	 * range of int64. No problem to do so since such large values
+	 * have no significant difference.
+	 */
+	if (restbytes > PG_INT64_MAX)
+		restbytes = PG_INT64_MAX;
+
+	return restbytes;
+}
+
 /*
  * Retreat *logSegNo to the last segment that we need to retain because of
  * either wal_keep_segments or replication slots.
@@ -9462,38 +9668,102 @@ CreateRestartPoint(int flags)
 static void
 KeepLogSeg(XLogRecPtr recptr, XLogSegNo *logSegNo)
 {
-	XLogSegNo	segno;
-	XLogRecPtr	keep;
+	static XLogSegNo last_lost_segs = 0;
+	static int last_nslots = 0;
+	static char *last_slot_name = NULL;
+	XLogRecPtr	slotminptr = InvalidXLogRecPtr;
+	XLogSegNo	minSegNo;
+	XLogSegNo	minSlotSegNo;
+	int			nslots_affected = 0;
 
-	XLByteToSeg(recptr, segno, wal_segment_size);
-	keep = XLogGetReplicationSlotMinimumLSN();
+	if (max_replication_slots > 0)
+		slotminptr = XLogGetReplicationSlotMinimumLSN();
 
-	/* compute limit for wal_keep_segments first */
-	if (wal_keep_segments > 0)
+	/*
+	 * We should keep certain number of WAL segments after this checkpoint.
+	 */
+	minSegNo = GetOldestKeepSegment(recptr, slotminptr);
+
+	/*
+	 * Warn the checkpoint is going to flush the segments required by
+	 * replication slots.
+	 */
+	if (!XLogRecPtrIsInvalid(slotminptr))
 	{
-		/* avoid underflow, don't go below 1 */
-		if (segno <= wal_keep_segments)
-			segno = 1;
-		else
-			segno = segno - wal_keep_segments;
+		Assert (max_replication_slots > 0);
+
+		XLByteToSeg(slotminptr, minSlotSegNo, wal_segment_size);
+
+		if (minSlotSegNo < minSegNo)
+		{
+			/* Some slots has lost required segments */
+			XLogSegNo	lost_segs = minSegNo - minSlotSegNo;
+			ReplicationSlot *earliest = NULL;
+			char	   *earliest_name = NULL;
+			int			i;
+
+			/* Find the most affected slot */
+			LWLockAcquire(ReplicationSlotControlLock, LW_SHARED);
+			for (i = 0 ; i < max_replication_slots ; i++)
+			{
+				ReplicationSlot *s =
+					&ReplicationSlotCtl->replication_slots[i];
+				XLogSegNo slotSegNo;
+
+				XLByteToSeg(s->data.restart_lsn, slotSegNo, wal_segment_size);
+
+				if (s->in_use && s->active_pid == 0 && slotSegNo < minSegNo)
+				{
+					nslots_affected++;
+
+					if (earliest == NULL ||
+						s->data.restart_lsn < earliest->data.restart_lsn)
+						earliest = s;
+				}
+			}
+
+			if (earliest)
+			{
+				MemoryContext oldcxt = MemoryContextSwitchTo(TopMemoryContext);
+				earliest_name = pstrdup(NameStr(earliest->data.name));
+				MemoryContextSwitchTo(oldcxt);
+			}
+
+			LWLockRelease(ReplicationSlotControlLock);
+
+			/* Emit WARNING if something has changed */
+			if (earliest_name &&
+				(last_lost_segs != lost_segs || last_nslots != nslots_affected))
+			{
+				ereport(WARNING,
+						(errmsg_plural ("%d replication slot has lost required WAL segments by %lu segments",
+										"%d replication slots have lost required WAL segments by %lu segments",
+										nslots_affected, nslots_affected,
+										lost_segs),
+						 errdetail("Most affected slot is %s.",
+								   earliest_name)));
+
+				if (last_slot_name)
+					pfree(last_slot_name);
+				last_slot_name = earliest_name;
+				last_lost_segs = lost_segs;
+				last_nslots = nslots_affected;
+			}
+		}
 	}
 
-	/* then check whether slots limit removal further */
-	if (max_replication_slots > 0 && keep != InvalidXLogRecPtr)
+	/* Reset the state if no affected slots remain. */
+	if (nslots_affected == 0 && last_slot_name)
 	{
-		XLogSegNo	slotSegNo;
-
-		XLByteToSeg(keep, slotSegNo, wal_segment_size);
-
-		if (slotSegNo <= 0)
-			segno = 1;
-		else if (slotSegNo < segno)
-			segno = slotSegNo;
+		pfree(last_slot_name);
+		last_slot_name = NULL;
+		last_lost_segs = 0;
+		last_nslots = 0;
 	}
 
 	/* don't delete WAL segments newer than the calculated segment */
-	if (segno < *logSegNo)
-		*logSegNo = segno;
+	if (minSegNo < *logSegNo)
+		*logSegNo = minSegNo;
 }
 
 /*
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index 83d00c6cde..775b8b7f20 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -863,7 +863,9 @@ CREATE VIEW pg_replication_slots AS
             L.xmin,
             L.catalog_xmin,
             L.restart_lsn,
-            L.confirmed_flush_lsn
+            L.confirmed_flush_lsn,
+            L.wal_status,
+            L.remain
     FROM pg_get_replication_slots() AS L
             LEFT JOIN pg_database D ON (L.datoid = D.oid);
 
diff --git a/src/backend/replication/slot.c b/src/backend/replication/slot.c
index d90c7235e9..a26f7999aa 100644
--- a/src/backend/replication/slot.c
+++ b/src/backend/replication/slot.c
@@ -49,6 +49,7 @@
 #include "storage/proc.h"
 #include "storage/procarray.h"
 #include "utils/builtins.h"
+#include "utils/memutils.h"
 
 /*
  * Replication slot on-disk data structure.
diff --git a/src/backend/replication/slotfuncs.c b/src/backend/replication/slotfuncs.c
index ce0c9127bc..47cd4375a1 100644
--- a/src/backend/replication/slotfuncs.c
+++ b/src/backend/replication/slotfuncs.c
@@ -234,7 +234,7 @@ pg_drop_replication_slot(PG_FUNCTION_ARGS)
 Datum
 pg_get_replication_slots(PG_FUNCTION_ARGS)
 {
-#define PG_GET_REPLICATION_SLOTS_COLS 11
+#define PG_GET_REPLICATION_SLOTS_COLS 13
 	ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo;
 	TupleDesc	tupdesc;
 	Tuplestorestate *tupstore;
@@ -288,6 +288,7 @@ pg_get_replication_slots(PG_FUNCTION_ARGS)
 		Oid			database;
 		NameData	slot_name;
 		NameData	plugin;
+		WalAvailability walstate;
 		int			i;
 
 		if (!slot->in_use)
@@ -355,6 +356,42 @@ pg_get_replication_slots(PG_FUNCTION_ARGS)
 		else
 			nulls[i++] = true;
 
+		walstate = GetWalAvailability(restart_lsn, active_pid);
+
+		switch (walstate)
+		{
+			case WALAVAIL_INVALID_LSN:
+				nulls[i++] = true;
+				break;
+
+			case WALAVAIL_NORMAL:
+				values[i++] = CStringGetTextDatum("normal");
+				break;
+
+			case WALAVAIL_PRESERVED:
+				values[i++] = CStringGetTextDatum("keeping");
+				break;
+
+			case WALAVAIL_BEING_REMOVED:
+				values[i++] = CStringGetTextDatum("losing");
+				break;
+
+			case WALAVAIL_REMOVED:
+				values[i++] = CStringGetTextDatum("lost");
+				break;
+		}
+
+		if (max_slot_wal_keep_size_mb >= 0 &&
+			(walstate == WALAVAIL_NORMAL ||
+			 walstate == WALAVAIL_PRESERVED))
+		{
+			values[i++] =
+				Int64GetDatum(DistanceToWalRemoval(GetXLogWriteRecPtr(),
+												   restart_lsn));
+		}
+		else
+			nulls[i++] = true;
+
 		tuplestore_putvalues(tupstore, tupdesc, values, nulls);
 	}
 	LWLockRelease(ReplicationSlotControlLock);
diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c
index 79bc7ac8ca..a4f0a4e0e3 100644
--- a/src/backend/utils/misc/guc.c
+++ b/src/backend/utils/misc/guc.c
@@ -2771,6 +2771,19 @@ static struct config_int ConfigureNamesInt[] =
 		NULL, NULL, NULL
 	},
 
+	{
+		{"max_slot_wal_keep_size", PGC_SIGHUP, REPLICATION_SENDING,
+			gettext_noop("Sets the maximum WAL size that can be reserved by replication slots."),
+			gettext_noop("Replication slots will be marked as failed, and segments released "
+						 "for deletion or recycling, if this much space is occupied by WAL "
+						 "on disk."),
+			GUC_UNIT_MB
+		},
+		&max_slot_wal_keep_size_mb,
+		-1, -1, MAX_KILOBYTES,
+		NULL, NULL, NULL
+	},
+
 	{
 		{"wal_sender_timeout", PGC_USERSET, REPLICATION_SENDING,
 			gettext_noop("Sets the maximum time to wait for WAL replication."),
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index e9f8ca775d..0b696e7044 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -287,6 +287,7 @@
 #max_wal_senders = 10		# max number of walsender processes
 				# (change requires restart)
 #wal_keep_segments = 0		# in logfile segments; 0 disables
+#max_slot_wal_keep_size = -1	# measured in bytes; -1 disables
 #wal_sender_timeout = 60s	# in milliseconds; 0 disables
 
 #max_replication_slots = 10	# max number of replication slots
diff --git a/src/include/access/xlog.h b/src/include/access/xlog.h
index 9ec7b31cce..9d29d2263f 100644
--- a/src/include/access/xlog.h
+++ b/src/include/access/xlog.h
@@ -108,6 +108,7 @@ extern int	wal_segment_size;
 extern int	min_wal_size_mb;
 extern int	max_wal_size_mb;
 extern int	wal_keep_segments;
+extern int	max_slot_wal_keep_size_mb;
 extern int	XLOGbuffers;
 extern int	XLogArchiveTimeout;
 extern int	wal_retrieve_retry_interval;
@@ -255,6 +256,20 @@ typedef struct CheckpointStatsData
 
 extern CheckpointStatsData CheckpointStats;
 
+/*
+ * WAL segment availability status
+ *
+ * This is used as the return value of GetWalAvailability.
+ */
+typedef enum WalAvailability
+{
+	WALAVAIL_INVALID_LSN,			/* parameter error */
+	WALAVAIL_NORMAL,				/* WAL segment is within max_wal_size */
+	WALAVAIL_PRESERVED,				/* WAL segment is preserved by repslots */
+	WALAVAIL_BEING_REMOVED,			/* WAL segment is no longer preserved */
+	WALAVAIL_REMOVED				/* WAL segment has been removed */
+} WalAvailability;
+
 struct XLogRecData;
 
 extern XLogRecPtr XLogInsertRecord(struct XLogRecData *rdata,
@@ -268,6 +283,7 @@ extern int	XLogFileOpen(XLogSegNo segno);
 
 extern void CheckXLogRemoved(XLogSegNo segno, TimeLineID tli);
 extern XLogSegNo XLogGetLastRemovedSegno(void);
+extern XLogSegNo FindOldestXLogFileSegNo(void);
 extern void XLogSetAsyncXactLSN(XLogRecPtr record);
 extern void XLogSetReplicationSlotMinimumLSN(XLogRecPtr lsn);
 
@@ -305,6 +321,9 @@ extern void ShutdownXLOG(int code, Datum arg);
 extern void InitXLOGAccess(void);
 extern void CreateCheckPoint(int flags);
 extern bool CreateRestartPoint(int flags);
+extern WalAvailability GetWalAvailability(XLogRecPtr restart_lsn,
+										  pid_t walsender_pid);
+extern int64 DistanceToWalRemoval(XLogRecPtr currLSN, XLogRecPtr targetLSN);
 extern void XLogPutNextOid(Oid nextOid);
 extern XLogRecPtr XLogRestorePoint(const char *rpName);
 extern void UpdateFullPageWrites(void);
diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat
index a6a708cca9..2025f34bfd 100644
--- a/src/include/catalog/pg_proc.dat
+++ b/src/include/catalog/pg_proc.dat
@@ -9971,9 +9971,9 @@
   proname => 'pg_get_replication_slots', prorows => '10', proisstrict => 'f',
   proretset => 't', provolatile => 's', prorettype => 'record',
   proargtypes => '',
-  proallargtypes => '{name,name,text,oid,bool,bool,int4,xid,xid,pg_lsn,pg_lsn}',
-  proargmodes => '{o,o,o,o,o,o,o,o,o,o,o}',
-  proargnames => '{slot_name,plugin,slot_type,datoid,temporary,active,active_pid,xmin,catalog_xmin,restart_lsn,confirmed_flush_lsn}',
+  proallargtypes => '{name,name,text,oid,bool,bool,int4,xid,xid,pg_lsn,pg_lsn,text,int8}',
+  proargmodes => '{o,o,o,o,o,o,o,o,o,o,o,o,o}',
+  proargnames => '{slot_name,plugin,slot_type,datoid,temporary,active,active_pid,xmin,catalog_xmin,restart_lsn,confirmed_flush_lsn,wal_status,remain}',
   prosrc => 'pg_get_replication_slots' },
 { oid => '3786', descr => 'set up a logical replication slot',
   proname => 'pg_create_logical_replication_slot', provolatile => 'v',
diff --git a/src/test/recovery/t/018_replslot_limit.pl b/src/test/recovery/t/018_replslot_limit.pl
new file mode 100644
index 0000000000..39a8780b0f
--- /dev/null
+++ b/src/test/recovery/t/018_replslot_limit.pl
@@ -0,0 +1,203 @@
+# Test for replication slot limit
+# Ensure that max_slot_wal_keep_size limits the number of WAL files to
+# be kept by replication slots.
+
+use strict;
+use warnings;
+use File::Path qw(rmtree);
+use PostgresNode;
+use TestLib;
+use Test::More tests => 13;
+use Time::HiRes qw(usleep);
+
+$ENV{PGDATABASE} = 'postgres';
+
+# Initialize master node, setting wal-segsize to 1MB
+my $node_master = get_new_node('master');
+$node_master->init(allows_streaming => 1, extra => ['--wal-segsize=1']);
+$node_master->append_conf('postgresql.conf', qq(
+min_wal_size = 2MB
+max_wal_size = 4MB
+log_checkpoints = yes
+));
+$node_master->start;
+$node_master->safe_psql('postgres', "SELECT pg_create_physical_replication_slot('rep1')");
+
+# The slot state and remain should be null before the first connection
+my $result = $node_master->safe_psql('postgres', "SELECT restart_lsn is NULL, wal_status is NULL, remain is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'");
+is($result, "t|t|t", 'check the state of non-reserved slot is "unknown"');
+
+
+# Take backup
+my $backup_name = 'my_backup';
+$node_master->backup($backup_name);
+
+# Create a standby linking to it using the replication slot
+my $node_standby = get_new_node('standby_1');
+$node_standby->init_from_backup($node_master, $backup_name, has_streaming => 1);
+$node_standby->append_conf('postgresql.conf', "primary_slot_name = 'rep1'");
+
+$node_standby->start;
+
+# Wait until standby has replayed enough data
+my $start_lsn = $node_master->lsn('write');
+$node_master->wait_for_catchup($node_standby, 'replay', $start_lsn);
+
+# Stop standby
+$node_standby->stop;
+
+
+# Preparation done, the slot is the state "normal" now
+$result = $node_master->safe_psql('postgres', "SELECT restart_lsn, wal_status, remain is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'");
+is($result, "$start_lsn|normal|t", 'check the catching-up state');
+
+# Advance WAL by five segments (= 5MB) on master
+advance_wal($node_master, 1);
+$node_master->safe_psql('postgres', "CHECKPOINT;");
+
+# The slot is always "safe" when fitting max_wal_size
+$result = $node_master->safe_psql('postgres', "SELECT restart_lsn, wal_status, remain is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'");
+is($result, "$start_lsn|normal|t", 'check that restart_lsn is in max_wal_size');
+
+advance_wal($node_master, 4);
+$node_master->safe_psql('postgres', "CHECKPOINT;");
+
+# The slot is always "safe" when max_slot_wal_keep_size is not set
+$result = $node_master->safe_psql('postgres', "SELECT restart_lsn, wal_status, remain is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'");
+is($result, "$start_lsn|normal|t", 'check that slot is working');
+
+# The standby can reconnect to master
+$node_standby->start;
+
+$start_lsn = $node_master->lsn('write');
+$node_master->wait_for_catchup($node_standby, 'replay', $start_lsn);
+
+$node_standby->stop;
+
+# Set max_slot_wal_keep_size on master
+my $max_slot_wal_keep_size_mb = 6;
+$node_master->append_conf('postgresql.conf', qq(
+max_slot_wal_keep_size = ${max_slot_wal_keep_size_mb}MB
+));
+$node_master->reload;
+
+# The slot is in safe state. The remaining bytes should be as almost
+# (max_slot_wal_keep_size + 1) times large as the segment size
+$result = $node_master->safe_psql('postgres', "SELECT restart_lsn, wal_status, pg_size_pretty(remain) as remain FROM pg_replication_slots WHERE slot_name = 'rep1'");
+is($result, "$start_lsn|normal|7168 kB", 'check that max_slot_wal_keep_size is working');
+
+# Advance WAL again then checkpoint, reducing remain by 2 MB.
+advance_wal($node_master, 2);
+$node_master->safe_psql('postgres', "CHECKPOINT;");
+
+# The slot is still working
+$result = $node_master->safe_psql('postgres', "SELECT restart_lsn, wal_status, pg_size_pretty(remain) as remain FROM pg_replication_slots WHERE slot_name = 'rep1'");
+is($result, "$start_lsn|normal|5120 kB", 'check that remaining byte is calculated correctly');
+
+# wal_keep_segments overrides max_slot_wal_keep_size
+$result = $node_master->safe_psql('postgres', "ALTER SYSTEM SET wal_keep_segments to 8; SELECT pg_reload_conf();");
+$result = $node_master->safe_psql('postgres', "SELECT restart_lsn, wal_status, pg_size_pretty(remain) as remain FROM pg_replication_slots WHERE slot_name = 'rep1'");
+is($result, "$start_lsn|normal|7168 kB", 'check that wal_keep_segments overrides max_slot_wal_keep_size');
+
+# restore wal_keep_segments
+$result = $node_master->safe_psql('postgres', "ALTER SYSTEM SET wal_keep_segments to 0; SELECT pg_reload_conf();");
+
+# Advance WAL again without checkpoint, reducing remain by 4 MB.
+advance_wal($node_master, 4);
+
+# Slot gets into 'keeping' state
+$result = $node_master->safe_psql('postgres', "SELECT restart_lsn, wal_status, pg_size_pretty(remain) as remain FROM pg_replication_slots WHERE slot_name = 'rep1'");
+is($result, "$start_lsn|keeping|1024 kB", 'check that the slot state changes to "keeping"');
+
+# do checkpoint so that the next checkpoint runs too early
+$node_master->safe_psql('postgres', "CHECKPOINT;");
+
+# Advance WAL again without checkpoint; remain goes to 0.
+advance_wal($node_master, 1);
+
+# Slot gets into 'lost' state
+$result = $node_master->safe_psql('postgres', "SELECT restart_lsn, wal_status, remain is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'");
+is($result, "$start_lsn|lost|t", 'check that the slot state changes to "lost"');
+
+# The standby still can connect to master before a checkpoint
+$node_standby->start;
+
+$start_lsn = $node_master->lsn('write');
+$node_master->wait_for_catchup($node_standby, 'replay', $start_lsn);
+
+$node_standby->stop;
+
+ok(!find_in_log($node_standby,
+				"requested WAL segment [0-9A-F]+ has already been removed"),
+   'check that required WAL segments are still available');
+
+# Advance WAL again, the slot loses the oldest segment.
+my $logstart = get_log_size($node_master);
+advance_wal($node_master, 7);
+$node_master->safe_psql('postgres', "CHECKPOINT;");
+
+# WARNING should be issued
+ok(find_in_log($node_master,
+			   "1 replication slot has lost required WAL segments by 1 segments\n".
+			   ".*Most affected slot is rep1.",
+			   $logstart),
+   'check that the warning is logged');
+
+# This slot should be broken
+$result = $node_master->safe_psql('postgres', "SELECT restart_lsn, wal_status, remain is NULL FROM pg_replication_slots WHERE slot_name = 'rep1'");
+is($result, "$start_lsn|lost|t", 'check that the slot state changes to "lost"');
+
+# The standby no longer can connect to the master
+$logstart = get_log_size($node_standby);
+$node_standby->start;
+
+my $failed = 0;
+for (my $i = 0 ; $i < 10000 ; $i++)
+{
+	if (find_in_log($node_standby,
+					"requested WAL segment [0-9A-F]+ has already been removed",
+					$logstart))
+	{
+		$failed = 1;
+		last;
+	}
+	usleep(100_000);
+}
+ok($failed, 'check that replication has been broken');
+
+$node_standby->stop;
+
+#####################################
+# Advance WAL of $node by $n segments
+sub advance_wal
+{
+	my ($node, $n) = @_;
+
+	# Advance by $n segments (= (16 * $n) MB) on master
+	for (my $i = 0 ; $i < $n ; $i++)
+	{
+		$node->safe_psql('postgres', "CREATE TABLE t (); DROP TABLE t; SELECT pg_switch_wal();");
+	}
+}
+
+# return the size of logfile of $node in bytes
+sub get_log_size
+{
+	my ($node) = @_;
+
+	return (stat $node->logfile)[7];
+}
+
+# find $pat in logfile of $node after $off-th byte
+sub find_in_log
+{
+	my ($node, $pat, $off) = @_;
+
+	$off = 0 unless defined $off;
+	my $log = TestLib::slurp_file($node->logfile);
+	return 0 if (length($log) <= $off);
+
+	$log = substr($log, $off);
+
+	return $log =~ m/$pat/;
+}
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index 7245b0e13b..8688f7138f 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1462,8 +1462,10 @@ pg_replication_slots| SELECT l.slot_name,
     l.xmin,
     l.catalog_xmin,
     l.restart_lsn,
-    l.confirmed_flush_lsn
-   FROM (pg_get_replication_slots() l(slot_name, plugin, slot_type, datoid, temporary, active, active_pid, xmin, catalog_xmin, restart_lsn, confirmed_flush_lsn)
+    l.confirmed_flush_lsn,
+    l.wal_status,
+    l.remain
+   FROM (pg_get_replication_slots() l(slot_name, plugin, slot_type, datoid, temporary, active, active_pid, xmin, catalog_xmin, restart_lsn, confirmed_flush_lsn, wal_status, remain)
      LEFT JOIN pg_database d ON ((l.datoid = d.oid)));
 pg_roles| SELECT pg_authid.rolname,
     pg_authid.rolsuper,
-- 
2.20.1


--T4sUOijqQbZv57TR
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v21-0002-Check-removal-of-in-reading-segment-file.patch"



^ permalink  raw  reply  [nested|flat] 125+ messages in thread

* [PATCH v24 01/15] Add a syntax to create Incrementally Maintainable Materialized Views
@ 2019-12-20 01:05 Yugo Nagata <[email protected]>
  0 siblings, 0 replies; 125+ messages in thread

From: Yugo Nagata @ 2019-12-20 01:05 UTC (permalink / raw)

Allow to create Incrementally Maintainable Materialized View (IMMV)
by using INCREMENTAL option in CREATE MATERIALIZED VIEW command
as follow:

     CREATE [INCREMANTAL] MATERIALIZED VIEW xxxxx AS SELECT ....;
---
 src/backend/nodes/copyfuncs.c  |  1 +
 src/backend/nodes/equalfuncs.c |  1 +
 src/backend/nodes/outfuncs.c   |  1 +
 src/backend/nodes/readfuncs.c  |  1 +
 src/backend/parser/gram.y      | 32 +++++++++++++++++++++-----------
 src/include/nodes/primnodes.h  |  1 +
 src/include/parser/kwlist.h    |  1 +
 7 files changed, 27 insertions(+), 11 deletions(-)

diff --git a/src/backend/nodes/copyfuncs.c b/src/backend/nodes/copyfuncs.c
index 82464c9889..5769a10586 100644
--- a/src/backend/nodes/copyfuncs.c
+++ b/src/backend/nodes/copyfuncs.c
@@ -1423,6 +1423,7 @@ _copyIntoClause(const IntoClause *from)
 	COPY_STRING_FIELD(tableSpaceName);
 	COPY_NODE_FIELD(viewQuery);
 	COPY_SCALAR_FIELD(skipData);
+	COPY_SCALAR_FIELD(ivm);
 
 	return newnode;
 }
diff --git a/src/backend/nodes/equalfuncs.c b/src/backend/nodes/equalfuncs.c
index f537d3eb96..80497dc240 100644
--- a/src/backend/nodes/equalfuncs.c
+++ b/src/backend/nodes/equalfuncs.c
@@ -155,6 +155,7 @@ _equalIntoClause(const IntoClause *a, const IntoClause *b)
 	COMPARE_STRING_FIELD(tableSpaceName);
 	COMPARE_NODE_FIELD(viewQuery);
 	COMPARE_SCALAR_FIELD(skipData);
+	COMPARE_SCALAR_FIELD(ivm);
 
 	return true;
 }
diff --git a/src/backend/nodes/outfuncs.c b/src/backend/nodes/outfuncs.c
index 2e5ed77e18..f6166f7859 100644
--- a/src/backend/nodes/outfuncs.c
+++ b/src/backend/nodes/outfuncs.c
@@ -1116,6 +1116,7 @@ _outIntoClause(StringInfo str, const IntoClause *node)
 	WRITE_STRING_FIELD(tableSpaceName);
 	WRITE_NODE_FIELD(viewQuery);
 	WRITE_BOOL_FIELD(skipData);
+	WRITE_BOOL_FIELD(ivm);
 }
 
 static void
diff --git a/src/backend/nodes/readfuncs.c b/src/backend/nodes/readfuncs.c
index abf08b7a2f..1e55a58f69 100644
--- a/src/backend/nodes/readfuncs.c
+++ b/src/backend/nodes/readfuncs.c
@@ -565,6 +565,7 @@ _readIntoClause(void)
 	READ_STRING_FIELD(tableSpaceName);
 	READ_NODE_FIELD(viewQuery);
 	READ_BOOL_FIELD(skipData);
+	READ_BOOL_FIELD(ivm);
 
 	READ_DONE();
 }
diff --git a/src/backend/parser/gram.y b/src/backend/parser/gram.y
index d0eb80e69c..78e17a655b 100644
--- a/src/backend/parser/gram.y
+++ b/src/backend/parser/gram.y
@@ -444,6 +444,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
 
 %type <range>	OptTempTableName
 %type <into>	into_clause create_as_target create_mv_target
+%type <boolean>	incremental
 
 %type <defelt>	createfunc_opt_item common_func_opt_item dostmt_opt_item
 %type <fun_param> func_arg func_arg_with_default table_func_column aggr_arg
@@ -675,7 +676,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
 	HANDLER HAVING HEADER_P HOLD HOUR_P
 
 	IDENTITY_P IF_P ILIKE IMMEDIATE IMMUTABLE IMPLICIT_P IMPORT_P IN_P INCLUDE
-	INCLUDING INCREMENT INDEX INDEXES INHERIT INHERITS INITIALLY INLINE_P
+	INCLUDING INCREMENT INCREMENTAL INDEX INDEXES INHERIT INHERITS INITIALLY INLINE_P
 	INNER_P INOUT INPUT_P INSENSITIVE INSERT INSTEAD INT_P INTEGER
 	INTERSECT INTERVAL INTO INVOKER IS ISNULL ISOLATION
 
@@ -4271,30 +4272,32 @@ opt_with_data:
  *****************************************************************************/
 
 CreateMatViewStmt:
-		CREATE OptNoLog MATERIALIZED VIEW create_mv_target AS SelectStmt opt_with_data
+		CREATE OptNoLog incremental MATERIALIZED VIEW create_mv_target AS SelectStmt opt_with_data
 				{
 					CreateTableAsStmt *ctas = makeNode(CreateTableAsStmt);
-					ctas->query = $7;
-					ctas->into = $5;
+					ctas->query = $8;
+					ctas->into = $6;
 					ctas->objtype = OBJECT_MATVIEW;
 					ctas->is_select_into = false;
 					ctas->if_not_exists = false;
 					/* cram additional flags into the IntoClause */
-					$5->rel->relpersistence = $2;
-					$5->skipData = !($8);
+					$6->rel->relpersistence = $2;
+					$6->skipData = !($9);
+					$6->ivm = $3;
 					$$ = (Node *) ctas;
 				}
-		| CREATE OptNoLog MATERIALIZED VIEW IF_P NOT EXISTS create_mv_target AS SelectStmt opt_with_data
+		| CREATE OptNoLog incremental MATERIALIZED VIEW IF_P NOT EXISTS create_mv_target AS SelectStmt opt_with_data
 				{
 					CreateTableAsStmt *ctas = makeNode(CreateTableAsStmt);
-					ctas->query = $10;
-					ctas->into = $8;
+					ctas->query = $11;
+					ctas->into = $9;
 					ctas->objtype = OBJECT_MATVIEW;
 					ctas->is_select_into = false;
 					ctas->if_not_exists = true;
 					/* cram additional flags into the IntoClause */
-					$8->rel->relpersistence = $2;
-					$8->skipData = !($11);
+					$9->rel->relpersistence = $2;
+					$9->skipData = !($12);
+					$9->ivm = $3;
 					$$ = (Node *) ctas;
 				}
 		;
@@ -4311,9 +4314,14 @@ create_mv_target:
 					$$->tableSpaceName = $5;
 					$$->viewQuery = NULL;		/* filled at analysis time */
 					$$->skipData = false;		/* might get changed later */
+					$$->ivm = false;
 				}
 		;
 
+incremental:	INCREMENTAL				{ $$ = true; }
+				| /*EMPTY*/				{ $$ = false; }
+		;
+
 OptNoLog:	UNLOGGED					{ $$ = RELPERSISTENCE_UNLOGGED; }
 			| /*EMPTY*/					{ $$ = RELPERSISTENCE_PERMANENT; }
 		;
@@ -15664,6 +15672,7 @@ unreserved_keyword:
 			| INCLUDE
 			| INCLUDING
 			| INCREMENT
+			| INCREMENTAL
 			| INDEX
 			| INDEXES
 			| INHERIT
@@ -16216,6 +16225,7 @@ bare_label_keyword:
 			| INCLUDE
 			| INCLUDING
 			| INCREMENT
+			| INCREMENTAL
 			| INDEX
 			| INDEXES
 			| INHERIT
diff --git a/src/include/nodes/primnodes.h b/src/include/nodes/primnodes.h
index 433437643e..7ea80c7ded 100644
--- a/src/include/nodes/primnodes.h
+++ b/src/include/nodes/primnodes.h
@@ -117,6 +117,7 @@ typedef struct IntoClause
 	char	   *tableSpaceName; /* table space to use, or NULL */
 	Node	   *viewQuery;		/* materialized view's SELECT query */
 	bool		skipData;		/* true for WITH NO DATA */
+	bool		ivm;			/* true for WITH IVM */
 } IntoClause;
 
 
diff --git a/src/include/parser/kwlist.h b/src/include/parser/kwlist.h
index f836acf876..2cafb4e7fe 100644
--- a/src/include/parser/kwlist.h
+++ b/src/include/parser/kwlist.h
@@ -205,6 +205,7 @@ PG_KEYWORD("in", IN_P, RESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("include", INCLUDE, UNRESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("including", INCLUDING, UNRESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("increment", INCREMENT, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("incremental", INCREMENTAL, UNRESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("index", INDEX, UNRESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("indexes", INDEXES, UNRESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("inherit", INHERIT, UNRESERVED_KEYWORD, BARE_LABEL)
-- 
2.17.1


--Multipart=_Fri__29_Oct_2021_18_16_28_+0900_jlYRKjywLqhZ7oyk
Content-Type: text/x-diff;
 name="v24-0002-Add-relisivm-column-to-pg_class-system-catalog.patch"
Content-Disposition: attachment;
 filename="v24-0002-Add-relisivm-column-to-pg_class-system-catalog.patch"
Content-Transfer-Encoding: 7bit



^ permalink  raw  reply  [nested|flat] 125+ messages in thread

* [PATCH v37 01/11] Add a syntax to create Incrementally Maintainable Materialized Views
@ 2019-12-20 01:05 Yugo Nagata <[email protected]>
  0 siblings, 0 replies; 125+ messages in thread

From: Yugo Nagata @ 2019-12-20 01:05 UTC (permalink / raw)

Allow to create Incrementally Maintainable Materialized View (IMMV)
by using INCREMENTAL option in CREATE MATERIALIZED VIEW command
as follow:

     CREATE [INCREMANTAL] MATERIALIZED VIEW xxxxx AS SELECT ....;
---
 src/backend/parser/gram.y     | 32 +++++++++++++++++++++-----------
 src/include/nodes/primnodes.h |  1 +
 src/include/parser/kwlist.h   |  1 +
 3 files changed, 23 insertions(+), 11 deletions(-)

diff --git a/src/backend/parser/gram.y b/src/backend/parser/gram.y
index ff4e1388c55..9a353550c37 100644
--- a/src/backend/parser/gram.y
+++ b/src/backend/parser/gram.y
@@ -473,6 +473,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
 
 %type <range>	OptTempTableName
 %type <into>	into_clause create_as_target create_mv_target
+%type <boolean>	incremental
 
 %type <defelt>	createfunc_opt_item common_func_opt_item dostmt_opt_item
 %type <fun_param> func_arg func_arg_with_default table_func_column aggr_arg
@@ -776,7 +777,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
 	HANDLER HAVING HEADER_P HOLD HOUR_P
 
 	IDENTITY_P IF_P IGNORE_P ILIKE IMMEDIATE IMMUTABLE IMPLICIT_P IMPORT_P IN_P INCLUDE
-	INCLUDING INCREMENT INDENT INDEX INDEXES INHERIT INHERITS INITIALLY INLINE_P
+	INCLUDING INCREMENT INCREMENTAL INDENT INDEX INDEXES INHERIT INHERITS INITIALLY INLINE_P
 	INNER_P INOUT INPUT_P INSENSITIVE INSERT INSTEAD INT_P INTEGER
 	INTERSECT INTERVAL INTO INVOKER IS ISNULL ISOLATION
 
@@ -5039,32 +5040,34 @@ opt_with_data:
  *****************************************************************************/
 
 CreateMatViewStmt:
-		CREATE OptNoLog MATERIALIZED VIEW create_mv_target AS SelectStmt opt_with_data
+		CREATE OptNoLog incremental MATERIALIZED VIEW create_mv_target AS SelectStmt opt_with_data
 				{
 					CreateTableAsStmt *ctas = makeNode(CreateTableAsStmt);
 
-					ctas->query = $7;
-					ctas->into = $5;
+					ctas->query = $8;
+					ctas->into = $6;
 					ctas->objtype = OBJECT_MATVIEW;
 					ctas->is_select_into = false;
 					ctas->if_not_exists = false;
 					/* cram additional flags into the IntoClause */
-					$5->rel->relpersistence = $2;
-					$5->skipData = !($8);
+					$6->rel->relpersistence = $2;
+					$6->skipData = !($9);
+					$6->ivm = $3;
 					$$ = (Node *) ctas;
 				}
-		| CREATE OptNoLog MATERIALIZED VIEW IF_P NOT EXISTS create_mv_target AS SelectStmt opt_with_data
+		| CREATE OptNoLog incremental MATERIALIZED VIEW IF_P NOT EXISTS create_mv_target AS SelectStmt opt_with_data
 				{
 					CreateTableAsStmt *ctas = makeNode(CreateTableAsStmt);
 
-					ctas->query = $10;
-					ctas->into = $8;
+					ctas->query = $11;
+					ctas->into = $9;
 					ctas->objtype = OBJECT_MATVIEW;
 					ctas->is_select_into = false;
 					ctas->if_not_exists = true;
 					/* cram additional flags into the IntoClause */
-					$8->rel->relpersistence = $2;
-					$8->skipData = !($11);
+					$9->rel->relpersistence = $2;
+					$9->skipData = !($12);
+					$9->ivm = $3;
 					$$ = (Node *) ctas;
 				}
 		;
@@ -5081,9 +5084,14 @@ create_mv_target:
 					$$->tableSpaceName = $5;
 					$$->viewQuery = NULL;		/* filled at analysis time */
 					$$->skipData = false;		/* might get changed later */
+					$$->ivm = false;
 				}
 		;
 
+incremental:	INCREMENTAL				{ $$ = true; }
+				| /*EMPTY*/				{ $$ = false; }
+		;
+
 OptNoLog:	UNLOGGED					{ $$ = RELPERSISTENCE_UNLOGGED; }
 			| /*EMPTY*/					{ $$ = RELPERSISTENCE_PERMANENT; }
 		;
@@ -18948,6 +18956,7 @@ unreserved_keyword:
 			| INCLUDE
 			| INCLUDING
 			| INCREMENT
+			| INCREMENTAL
 			| INDENT
 			| INDEX
 			| INDEXES
@@ -19554,6 +19563,7 @@ bare_label_keyword:
 			| INCLUDE
 			| INCLUDING
 			| INCREMENT
+			| INCREMENTAL
 			| INDENT
 			| INDEX
 			| INDEXES
diff --git a/src/include/nodes/primnodes.h b/src/include/nodes/primnodes.h
index 7977ee24783..213adeec2e2 100644
--- a/src/include/nodes/primnodes.h
+++ b/src/include/nodes/primnodes.h
@@ -170,6 +170,7 @@ typedef struct IntoClause
 	/* materialized view's SELECT query */
 	struct Query *viewQuery pg_node_attr(query_jumble_ignore);
 	bool		skipData;		/* true for WITH NO DATA */
+	bool		ivm;			/* true for WITH IVM */
 } IntoClause;
 
 
diff --git a/src/include/parser/kwlist.h b/src/include/parser/kwlist.h
index 51ead54f015..7849ee7f960 100644
--- a/src/include/parser/kwlist.h
+++ b/src/include/parser/kwlist.h
@@ -216,6 +216,7 @@ PG_KEYWORD("in", IN_P, RESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("include", INCLUDE, UNRESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("including", INCLUDING, UNRESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("increment", INCREMENT, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("incremental", INCREMENTAL, UNRESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("indent", INDENT, UNRESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("index", INDEX, UNRESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("indexes", INDEXES, UNRESERVED_KEYWORD, BARE_LABEL)
-- 
2.43.0


--Multipart=_Fri__29_May_2026_23_14_17_+0900_Te0o73X2VqYK57Gd--





^ permalink  raw  reply  [nested|flat] 125+ messages in thread

* [PATCH v29 01/11] Add a syntax to create Incrementally Maintainable Materialized Views
@ 2019-12-20 01:05 Yugo Nagata <[email protected]>
  0 siblings, 0 replies; 125+ messages in thread

From: Yugo Nagata @ 2019-12-20 01:05 UTC (permalink / raw)

Allow to create Incrementally Maintainable Materialized View (IMMV)
by using INCREMENTAL option in CREATE MATERIALIZED VIEW command
as follow:

     CREATE [INCREMANTAL] MATERIALIZED VIEW xxxxx AS SELECT ....;
---
 src/backend/parser/gram.y     | 32 +++++++++++++++++++++-----------
 src/include/nodes/primnodes.h |  1 +
 src/include/parser/kwlist.h   |  1 +
 3 files changed, 23 insertions(+), 11 deletions(-)

diff --git a/src/backend/parser/gram.y b/src/backend/parser/gram.y
index 7d2032885e..33c647b0c7 100644
--- a/src/backend/parser/gram.y
+++ b/src/backend/parser/gram.y
@@ -465,6 +465,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
 
 %type <range>	OptTempTableName
 %type <into>	into_clause create_as_target create_mv_target
+%type <boolean>	incremental
 
 %type <defelt>	createfunc_opt_item common_func_opt_item dostmt_opt_item
 %type <fun_param> func_arg func_arg_with_default table_func_column aggr_arg
@@ -718,7 +719,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
 	HANDLER HAVING HEADER_P HOLD HOUR_P
 
 	IDENTITY_P IF_P ILIKE IMMEDIATE IMMUTABLE IMPLICIT_P IMPORT_P IN_P INCLUDE
-	INCLUDING INCREMENT INDENT INDEX INDEXES INHERIT INHERITS INITIALLY INLINE_P
+	INCLUDING INCREMENT INCREMENTAL INDENT INDEX INDEXES INHERIT INHERITS INITIALLY INLINE_P
 	INNER_P INOUT INPUT_P INSENSITIVE INSERT INSTEAD INT_P INTEGER
 	INTERSECT INTERVAL INTO INVOKER IS ISNULL ISOLATION
 
@@ -4652,32 +4653,34 @@ opt_with_data:
  *****************************************************************************/
 
 CreateMatViewStmt:
-		CREATE OptNoLog MATERIALIZED VIEW create_mv_target AS SelectStmt opt_with_data
+		CREATE OptNoLog incremental MATERIALIZED VIEW create_mv_target AS SelectStmt opt_with_data
 				{
 					CreateTableAsStmt *ctas = makeNode(CreateTableAsStmt);
 
-					ctas->query = $7;
-					ctas->into = $5;
+					ctas->query = $8;
+					ctas->into = $6;
 					ctas->objtype = OBJECT_MATVIEW;
 					ctas->is_select_into = false;
 					ctas->if_not_exists = false;
 					/* cram additional flags into the IntoClause */
-					$5->rel->relpersistence = $2;
-					$5->skipData = !($8);
+					$6->rel->relpersistence = $2;
+					$6->skipData = !($9);
+					$6->ivm = $3;
 					$$ = (Node *) ctas;
 				}
-		| CREATE OptNoLog MATERIALIZED VIEW IF_P NOT EXISTS create_mv_target AS SelectStmt opt_with_data
+		| CREATE OptNoLog incremental MATERIALIZED VIEW IF_P NOT EXISTS create_mv_target AS SelectStmt opt_with_data
 				{
 					CreateTableAsStmt *ctas = makeNode(CreateTableAsStmt);
 
-					ctas->query = $10;
-					ctas->into = $8;
+					ctas->query = $11;
+					ctas->into = $9;
 					ctas->objtype = OBJECT_MATVIEW;
 					ctas->is_select_into = false;
 					ctas->if_not_exists = true;
 					/* cram additional flags into the IntoClause */
-					$8->rel->relpersistence = $2;
-					$8->skipData = !($11);
+					$9->rel->relpersistence = $2;
+					$9->skipData = !($12);
+					$9->ivm = $3;
 					$$ = (Node *) ctas;
 				}
 		;
@@ -4694,9 +4697,14 @@ create_mv_target:
 					$$->tableSpaceName = $5;
 					$$->viewQuery = NULL;		/* filled at analysis time */
 					$$->skipData = false;		/* might get changed later */
+					$$->ivm = false;
 				}
 		;
 
+incremental:	INCREMENTAL				{ $$ = true; }
+				| /*EMPTY*/				{ $$ = false; }
+		;
+
 OptNoLog:	UNLOGGED					{ $$ = RELPERSISTENCE_UNLOGGED; }
 			| /*EMPTY*/					{ $$ = RELPERSISTENCE_PERMANENT; }
 		;
@@ -17141,6 +17149,7 @@ unreserved_keyword:
 			| INCLUDE
 			| INCLUDING
 			| INCREMENT
+			| INCREMENTAL
 			| INDENT
 			| INDEX
 			| INDEXES
@@ -17709,6 +17718,7 @@ bare_label_keyword:
 			| INCLUDE
 			| INCLUDING
 			| INCREMENT
+			| INCREMENTAL
 			| INDENT
 			| INDEX
 			| INDEXES
diff --git a/src/include/nodes/primnodes.h b/src/include/nodes/primnodes.h
index 60d72a876b..cecb968b36 100644
--- a/src/include/nodes/primnodes.h
+++ b/src/include/nodes/primnodes.h
@@ -146,6 +146,7 @@ typedef struct IntoClause
 	/* materialized view's SELECT query */
 	Node	   *viewQuery pg_node_attr(query_jumble_ignore);
 	bool		skipData;		/* true for WITH NO DATA */
+	bool		ivm;			/* true for WITH IVM */
 } IntoClause;
 
 
diff --git a/src/include/parser/kwlist.h b/src/include/parser/kwlist.h
index 5984dcfa4b..d60eb98d65 100644
--- a/src/include/parser/kwlist.h
+++ b/src/include/parser/kwlist.h
@@ -207,6 +207,7 @@ PG_KEYWORD("in", IN_P, RESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("include", INCLUDE, UNRESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("including", INCLUDING, UNRESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("increment", INCREMENT, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("incremental", INCREMENTAL, UNRESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("indent", INDENT, UNRESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("index", INDEX, UNRESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("indexes", INDEXES, UNRESERVED_KEYWORD, BARE_LABEL)
-- 
2.25.1


--Multipart=_Mon__28_Aug_2023_11_52_52_+0900_hj6L5h176QaSGtg7
Content-Type: text/x-diff;
 name="v29-0002-Add-relisivm-column-to-pg_class-system-catalog.patch"
Content-Disposition: attachment;
 filename="v29-0002-Add-relisivm-column-to-pg_class-system-catalog.patch"
Content-Transfer-Encoding: 7bit



^ permalink  raw  reply  [nested|flat] 125+ messages in thread

* [PATCH v26 01/10] Add a syntax to create Incrementally Maintainable Materialized Views
@ 2019-12-20 01:05 Yugo Nagata <[email protected]>
  0 siblings, 0 replies; 125+ messages in thread

From: Yugo Nagata @ 2019-12-20 01:05 UTC (permalink / raw)

Allow to create Incrementally Maintainable Materialized View (IMMV)
by using INCREMENTAL option in CREATE MATERIALIZED VIEW command
as follow:

     CREATE [INCREMANTAL] MATERIALIZED VIEW xxxxx AS SELECT ....;
---
 src/backend/nodes/copyfuncs.c  |  1 +
 src/backend/nodes/equalfuncs.c |  1 +
 src/backend/nodes/outfuncs.c   |  1 +
 src/backend/nodes/readfuncs.c  |  1 +
 src/backend/parser/gram.y      | 32 +++++++++++++++++++++-----------
 src/include/nodes/primnodes.h  |  1 +
 src/include/parser/kwlist.h    |  1 +
 7 files changed, 27 insertions(+), 11 deletions(-)

diff --git a/src/backend/nodes/copyfuncs.c b/src/backend/nodes/copyfuncs.c
index d4f8455a2b..eb28657791 100644
--- a/src/backend/nodes/copyfuncs.c
+++ b/src/backend/nodes/copyfuncs.c
@@ -1426,6 +1426,7 @@ _copyIntoClause(const IntoClause *from)
 	COPY_STRING_FIELD(tableSpaceName);
 	COPY_NODE_FIELD(viewQuery);
 	COPY_SCALAR_FIELD(skipData);
+	COPY_SCALAR_FIELD(ivm);
 
 	return newnode;
 }
diff --git a/src/backend/nodes/equalfuncs.c b/src/backend/nodes/equalfuncs.c
index f1002afe7a..521a87a8ea 100644
--- a/src/backend/nodes/equalfuncs.c
+++ b/src/backend/nodes/equalfuncs.c
@@ -155,6 +155,7 @@ _equalIntoClause(const IntoClause *a, const IntoClause *b)
 	COMPARE_STRING_FIELD(tableSpaceName);
 	COMPARE_NODE_FIELD(viewQuery);
 	COMPARE_SCALAR_FIELD(skipData);
+	COMPARE_SCALAR_FIELD(ivm);
 
 	return true;
 }
diff --git a/src/backend/nodes/outfuncs.c b/src/backend/nodes/outfuncs.c
index 6bdad462c7..801c41b978 100644
--- a/src/backend/nodes/outfuncs.c
+++ b/src/backend/nodes/outfuncs.c
@@ -1119,6 +1119,7 @@ _outIntoClause(StringInfo str, const IntoClause *node)
 	WRITE_STRING_FIELD(tableSpaceName);
 	WRITE_NODE_FIELD(viewQuery);
 	WRITE_BOOL_FIELD(skipData);
+	WRITE_BOOL_FIELD(ivm);
 }
 
 static void
diff --git a/src/backend/nodes/readfuncs.c b/src/backend/nodes/readfuncs.c
index 3f68f7c18d..cc6dcb7220 100644
--- a/src/backend/nodes/readfuncs.c
+++ b/src/backend/nodes/readfuncs.c
@@ -563,6 +563,7 @@ _readIntoClause(void)
 	READ_STRING_FIELD(tableSpaceName);
 	READ_NODE_FIELD(viewQuery);
 	READ_BOOL_FIELD(skipData);
+	READ_BOOL_FIELD(ivm);
 
 	READ_DONE();
 }
diff --git a/src/backend/parser/gram.y b/src/backend/parser/gram.y
index a03b33b53b..e80ce50f82 100644
--- a/src/backend/parser/gram.y
+++ b/src/backend/parser/gram.y
@@ -459,6 +459,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
 
 %type <range>	OptTempTableName
 %type <into>	into_clause create_as_target create_mv_target
+%type <boolean>	incremental
 
 %type <defelt>	createfunc_opt_item common_func_opt_item dostmt_opt_item
 %type <fun_param> func_arg func_arg_with_default table_func_column aggr_arg
@@ -693,7 +694,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
 	HANDLER HAVING HEADER_P HOLD HOUR_P
 
 	IDENTITY_P IF_P ILIKE IMMEDIATE IMMUTABLE IMPLICIT_P IMPORT_P IN_P INCLUDE
-	INCLUDING INCREMENT INDEX INDEXES INHERIT INHERITS INITIALLY INLINE_P
+	INCLUDING INCREMENT INCREMENTAL INDEX INDEXES INHERIT INHERITS INITIALLY INLINE_P
 	INNER_P INOUT INPUT_P INSENSITIVE INSERT INSTEAD INT_P INTEGER
 	INTERSECT INTERVAL INTO INVOKER IS ISNULL ISOLATION
 
@@ -4368,30 +4369,32 @@ opt_with_data:
  *****************************************************************************/
 
 CreateMatViewStmt:
-		CREATE OptNoLog MATERIALIZED VIEW create_mv_target AS SelectStmt opt_with_data
+		CREATE OptNoLog incremental MATERIALIZED VIEW create_mv_target AS SelectStmt opt_with_data
 				{
 					CreateTableAsStmt *ctas = makeNode(CreateTableAsStmt);
-					ctas->query = $7;
-					ctas->into = $5;
+					ctas->query = $8;
+					ctas->into = $6;
 					ctas->objtype = OBJECT_MATVIEW;
 					ctas->is_select_into = false;
 					ctas->if_not_exists = false;
 					/* cram additional flags into the IntoClause */
-					$5->rel->relpersistence = $2;
-					$5->skipData = !($8);
+					$6->rel->relpersistence = $2;
+					$6->skipData = !($9);
+					$6->ivm = $3;
 					$$ = (Node *) ctas;
 				}
-		| CREATE OptNoLog MATERIALIZED VIEW IF_P NOT EXISTS create_mv_target AS SelectStmt opt_with_data
+		| CREATE OptNoLog incremental MATERIALIZED VIEW IF_P NOT EXISTS create_mv_target AS SelectStmt opt_with_data
 				{
 					CreateTableAsStmt *ctas = makeNode(CreateTableAsStmt);
-					ctas->query = $10;
-					ctas->into = $8;
+					ctas->query = $11;
+					ctas->into = $9;
 					ctas->objtype = OBJECT_MATVIEW;
 					ctas->is_select_into = false;
 					ctas->if_not_exists = true;
 					/* cram additional flags into the IntoClause */
-					$8->rel->relpersistence = $2;
-					$8->skipData = !($11);
+					$9->rel->relpersistence = $2;
+					$9->skipData = !($12);
+					$9->ivm = $3;
 					$$ = (Node *) ctas;
 				}
 		;
@@ -4408,9 +4411,14 @@ create_mv_target:
 					$$->tableSpaceName = $5;
 					$$->viewQuery = NULL;		/* filled at analysis time */
 					$$->skipData = false;		/* might get changed later */
+					$$->ivm = false;
 				}
 		;
 
+incremental:	INCREMENTAL				{ $$ = true; }
+				| /*EMPTY*/				{ $$ = false; }
+		;
+
 OptNoLog:	UNLOGGED					{ $$ = RELPERSISTENCE_UNLOGGED; }
 			| /*EMPTY*/					{ $$ = RELPERSISTENCE_PERMANENT; }
 		;
@@ -15787,6 +15795,7 @@ unreserved_keyword:
 			| INCLUDE
 			| INCLUDING
 			| INCREMENT
+			| INCREMENTAL
 			| INDEX
 			| INDEXES
 			| INHERIT
@@ -16339,6 +16348,7 @@ bare_label_keyword:
 			| INCLUDE
 			| INCLUDING
 			| INCREMENT
+			| INCREMENTAL
 			| INDEX
 			| INDEXES
 			| INHERIT
diff --git a/src/include/nodes/primnodes.h b/src/include/nodes/primnodes.h
index dab5c4ff5d..bc1bcbda13 100644
--- a/src/include/nodes/primnodes.h
+++ b/src/include/nodes/primnodes.h
@@ -117,6 +117,7 @@ typedef struct IntoClause
 	char	   *tableSpaceName; /* table space to use, or NULL */
 	Node	   *viewQuery;		/* materialized view's SELECT query */
 	bool		skipData;		/* true for WITH NO DATA */
+	bool		ivm;			/* true for WITH IVM */
 } IntoClause;
 
 
diff --git a/src/include/parser/kwlist.h b/src/include/parser/kwlist.h
index bcef7eed2f..a43af4a597 100644
--- a/src/include/parser/kwlist.h
+++ b/src/include/parser/kwlist.h
@@ -205,6 +205,7 @@ PG_KEYWORD("in", IN_P, RESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("include", INCLUDE, UNRESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("including", INCLUDING, UNRESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("increment", INCREMENT, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("incremental", INCREMENTAL, UNRESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("index", INDEX, UNRESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("indexes", INDEXES, UNRESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("inherit", INHERIT, UNRESERVED_KEYWORD, BARE_LABEL)
-- 
2.17.1


--Multipart=_Mon__14_Mar_2022_19_12_17_+0900_E9HVp8j5QFkIIek.--





^ permalink  raw  reply  [nested|flat] 125+ messages in thread

* [PATCH v30 01/11] Add a syntax to create Incrementally Maintainable Materialized Views
@ 2019-12-20 01:05 Yugo Nagata <[email protected]>
  0 siblings, 0 replies; 125+ messages in thread

From: Yugo Nagata @ 2019-12-20 01:05 UTC (permalink / raw)

Allow to create Incrementally Maintainable Materialized View (IMMV)
by using INCREMENTAL option in CREATE MATERIALIZED VIEW command
as follow:

     CREATE [INCREMANTAL] MATERIALIZED VIEW xxxxx AS SELECT ....;
---
 src/backend/parser/gram.y     | 32 +++++++++++++++++++++-----------
 src/include/nodes/primnodes.h |  1 +
 src/include/parser/kwlist.h   |  1 +
 3 files changed, 23 insertions(+), 11 deletions(-)

diff --git a/src/backend/parser/gram.y b/src/backend/parser/gram.y
index 130f7fc7c3..7ba972ae5d 100644
--- a/src/backend/parser/gram.y
+++ b/src/backend/parser/gram.y
@@ -466,6 +466,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
 
 %type <range>	OptTempTableName
 %type <into>	into_clause create_as_target create_mv_target
+%type <boolean>	incremental
 
 %type <defelt>	createfunc_opt_item common_func_opt_item dostmt_opt_item
 %type <fun_param> func_arg func_arg_with_default table_func_column aggr_arg
@@ -719,7 +720,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
 	HANDLER HAVING HEADER_P HOLD HOUR_P
 
 	IDENTITY_P IF_P ILIKE IMMEDIATE IMMUTABLE IMPLICIT_P IMPORT_P IN_P INCLUDE
-	INCLUDING INCREMENT INDENT INDEX INDEXES INHERIT INHERITS INITIALLY INLINE_P
+	INCLUDING INCREMENT INCREMENTAL INDENT INDEX INDEXES INHERIT INHERITS INITIALLY INLINE_P
 	INNER_P INOUT INPUT_P INSENSITIVE INSERT INSTEAD INT_P INTEGER
 	INTERSECT INTERVAL INTO INVOKER IS ISNULL ISOLATION
 
@@ -4696,32 +4697,34 @@ opt_with_data:
  *****************************************************************************/
 
 CreateMatViewStmt:
-		CREATE OptNoLog MATERIALIZED VIEW create_mv_target AS SelectStmt opt_with_data
+		CREATE OptNoLog incremental MATERIALIZED VIEW create_mv_target AS SelectStmt opt_with_data
 				{
 					CreateTableAsStmt *ctas = makeNode(CreateTableAsStmt);
 
-					ctas->query = $7;
-					ctas->into = $5;
+					ctas->query = $8;
+					ctas->into = $6;
 					ctas->objtype = OBJECT_MATVIEW;
 					ctas->is_select_into = false;
 					ctas->if_not_exists = false;
 					/* cram additional flags into the IntoClause */
-					$5->rel->relpersistence = $2;
-					$5->skipData = !($8);
+					$6->rel->relpersistence = $2;
+					$6->skipData = !($9);
+					$6->ivm = $3;
 					$$ = (Node *) ctas;
 				}
-		| CREATE OptNoLog MATERIALIZED VIEW IF_P NOT EXISTS create_mv_target AS SelectStmt opt_with_data
+		| CREATE OptNoLog incremental MATERIALIZED VIEW IF_P NOT EXISTS create_mv_target AS SelectStmt opt_with_data
 				{
 					CreateTableAsStmt *ctas = makeNode(CreateTableAsStmt);
 
-					ctas->query = $10;
-					ctas->into = $8;
+					ctas->query = $11;
+					ctas->into = $9;
 					ctas->objtype = OBJECT_MATVIEW;
 					ctas->is_select_into = false;
 					ctas->if_not_exists = true;
 					/* cram additional flags into the IntoClause */
-					$8->rel->relpersistence = $2;
-					$8->skipData = !($11);
+					$9->rel->relpersistence = $2;
+					$9->skipData = !($12);
+					$9->ivm = $3;
 					$$ = (Node *) ctas;
 				}
 		;
@@ -4738,9 +4741,14 @@ create_mv_target:
 					$$->tableSpaceName = $5;
 					$$->viewQuery = NULL;		/* filled at analysis time */
 					$$->skipData = false;		/* might get changed later */
+					$$->ivm = false;
 				}
 		;
 
+incremental:	INCREMENTAL				{ $$ = true; }
+				| /*EMPTY*/				{ $$ = false; }
+		;
+
 OptNoLog:	UNLOGGED					{ $$ = RELPERSISTENCE_UNLOGGED; }
 			| /*EMPTY*/					{ $$ = RELPERSISTENCE_PERMANENT; }
 		;
@@ -17252,6 +17260,7 @@ unreserved_keyword:
 			| INCLUDE
 			| INCLUDING
 			| INCREMENT
+			| INCREMENTAL
 			| INDENT
 			| INDEX
 			| INDEXES
@@ -17820,6 +17829,7 @@ bare_label_keyword:
 			| INCLUDE
 			| INCLUDING
 			| INCREMENT
+			| INCREMENTAL
 			| INDENT
 			| INDEX
 			| INDEXES
diff --git a/src/include/nodes/primnodes.h b/src/include/nodes/primnodes.h
index 4a154606d2..5d61964f37 100644
--- a/src/include/nodes/primnodes.h
+++ b/src/include/nodes/primnodes.h
@@ -154,6 +154,7 @@ typedef struct IntoClause
 	/* materialized view's SELECT query */
 	Node	   *viewQuery pg_node_attr(query_jumble_ignore);
 	bool		skipData;		/* true for WITH NO DATA */
+	bool		ivm;			/* true for WITH IVM */
 } IntoClause;
 
 
diff --git a/src/include/parser/kwlist.h b/src/include/parser/kwlist.h
index 2331acac09..ab92b54642 100644
--- a/src/include/parser/kwlist.h
+++ b/src/include/parser/kwlist.h
@@ -207,6 +207,7 @@ PG_KEYWORD("in", IN_P, RESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("include", INCLUDE, UNRESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("including", INCLUDING, UNRESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("increment", INCREMENT, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("incremental", INCREMENTAL, UNRESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("indent", INDENT, UNRESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("index", INDEX, UNRESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("indexes", INDEXES, UNRESERVED_KEYWORD, BARE_LABEL)
-- 
2.25.1


--Multipart=_Mon__4_Mar_2024_11_58_46_+0900_UaponF/qQhQrVCFt
Content-Type: text/x-diff;
 name="v30-0002-Add-relisivm-column-to-pg_class-system-catalog.patch"
Content-Disposition: attachment;
 filename="v30-0002-Add-relisivm-column-to-pg_class-system-catalog.patch"
Content-Transfer-Encoding: 7bit



^ permalink  raw  reply  [nested|flat] 125+ messages in thread

* [PATCH v38 01/11] Add a syntax to create Incrementally Maintainable Materialized Views
@ 2019-12-20 01:05 Yugo Nagata <[email protected]>
  0 siblings, 0 replies; 125+ messages in thread

From: Yugo Nagata @ 2019-12-20 01:05 UTC (permalink / raw)

Allow to create Incrementally Maintainable Materialized View (IMMV)
by using INCREMENTAL option in CREATE MATERIALIZED VIEW command
as follow:

     CREATE [INCREMANTAL] MATERIALIZED VIEW xxxxx AS SELECT ....;
---
 src/backend/parser/gram.y     | 32 +++++++++++++++++++++-----------
 src/include/nodes/primnodes.h |  1 +
 src/include/parser/kwlist.h   |  1 +
 3 files changed, 23 insertions(+), 11 deletions(-)

diff --git a/src/backend/parser/gram.y b/src/backend/parser/gram.y
index ff4e1388c55..9a353550c37 100644
--- a/src/backend/parser/gram.y
+++ b/src/backend/parser/gram.y
@@ -473,6 +473,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
 
 %type <range>	OptTempTableName
 %type <into>	into_clause create_as_target create_mv_target
+%type <boolean>	incremental
 
 %type <defelt>	createfunc_opt_item common_func_opt_item dostmt_opt_item
 %type <fun_param> func_arg func_arg_with_default table_func_column aggr_arg
@@ -776,7 +777,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
 	HANDLER HAVING HEADER_P HOLD HOUR_P
 
 	IDENTITY_P IF_P IGNORE_P ILIKE IMMEDIATE IMMUTABLE IMPLICIT_P IMPORT_P IN_P INCLUDE
-	INCLUDING INCREMENT INDENT INDEX INDEXES INHERIT INHERITS INITIALLY INLINE_P
+	INCLUDING INCREMENT INCREMENTAL INDENT INDEX INDEXES INHERIT INHERITS INITIALLY INLINE_P
 	INNER_P INOUT INPUT_P INSENSITIVE INSERT INSTEAD INT_P INTEGER
 	INTERSECT INTERVAL INTO INVOKER IS ISNULL ISOLATION
 
@@ -5039,32 +5040,34 @@ opt_with_data:
  *****************************************************************************/
 
 CreateMatViewStmt:
-		CREATE OptNoLog MATERIALIZED VIEW create_mv_target AS SelectStmt opt_with_data
+		CREATE OptNoLog incremental MATERIALIZED VIEW create_mv_target AS SelectStmt opt_with_data
 				{
 					CreateTableAsStmt *ctas = makeNode(CreateTableAsStmt);
 
-					ctas->query = $7;
-					ctas->into = $5;
+					ctas->query = $8;
+					ctas->into = $6;
 					ctas->objtype = OBJECT_MATVIEW;
 					ctas->is_select_into = false;
 					ctas->if_not_exists = false;
 					/* cram additional flags into the IntoClause */
-					$5->rel->relpersistence = $2;
-					$5->skipData = !($8);
+					$6->rel->relpersistence = $2;
+					$6->skipData = !($9);
+					$6->ivm = $3;
 					$$ = (Node *) ctas;
 				}
-		| CREATE OptNoLog MATERIALIZED VIEW IF_P NOT EXISTS create_mv_target AS SelectStmt opt_with_data
+		| CREATE OptNoLog incremental MATERIALIZED VIEW IF_P NOT EXISTS create_mv_target AS SelectStmt opt_with_data
 				{
 					CreateTableAsStmt *ctas = makeNode(CreateTableAsStmt);
 
-					ctas->query = $10;
-					ctas->into = $8;
+					ctas->query = $11;
+					ctas->into = $9;
 					ctas->objtype = OBJECT_MATVIEW;
 					ctas->is_select_into = false;
 					ctas->if_not_exists = true;
 					/* cram additional flags into the IntoClause */
-					$8->rel->relpersistence = $2;
-					$8->skipData = !($11);
+					$9->rel->relpersistence = $2;
+					$9->skipData = !($12);
+					$9->ivm = $3;
 					$$ = (Node *) ctas;
 				}
 		;
@@ -5081,9 +5084,14 @@ create_mv_target:
 					$$->tableSpaceName = $5;
 					$$->viewQuery = NULL;		/* filled at analysis time */
 					$$->skipData = false;		/* might get changed later */
+					$$->ivm = false;
 				}
 		;
 
+incremental:	INCREMENTAL				{ $$ = true; }
+				| /*EMPTY*/				{ $$ = false; }
+		;
+
 OptNoLog:	UNLOGGED					{ $$ = RELPERSISTENCE_UNLOGGED; }
 			| /*EMPTY*/					{ $$ = RELPERSISTENCE_PERMANENT; }
 		;
@@ -18948,6 +18956,7 @@ unreserved_keyword:
 			| INCLUDE
 			| INCLUDING
 			| INCREMENT
+			| INCREMENTAL
 			| INDENT
 			| INDEX
 			| INDEXES
@@ -19554,6 +19563,7 @@ bare_label_keyword:
 			| INCLUDE
 			| INCLUDING
 			| INCREMENT
+			| INCREMENTAL
 			| INDENT
 			| INDEX
 			| INDEXES
diff --git a/src/include/nodes/primnodes.h b/src/include/nodes/primnodes.h
index bb05aeebee4..da35b15cc27 100644
--- a/src/include/nodes/primnodes.h
+++ b/src/include/nodes/primnodes.h
@@ -170,6 +170,7 @@ typedef struct IntoClause
 	/* materialized view's SELECT query */
 	struct Query *viewQuery pg_node_attr(query_jumble_ignore);
 	bool		skipData;		/* true for WITH NO DATA */
+	bool		ivm;			/* true for WITH IVM */
 } IntoClause;
 
 
diff --git a/src/include/parser/kwlist.h b/src/include/parser/kwlist.h
index 51ead54f015..7849ee7f960 100644
--- a/src/include/parser/kwlist.h
+++ b/src/include/parser/kwlist.h
@@ -216,6 +216,7 @@ PG_KEYWORD("in", IN_P, RESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("include", INCLUDE, UNRESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("including", INCLUDING, UNRESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("increment", INCREMENT, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("incremental", INCREMENTAL, UNRESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("indent", INDENT, UNRESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("index", INDEX, UNRESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("indexes", INDEXES, UNRESERVED_KEYWORD, BARE_LABEL)
-- 
2.43.0


--Multipart=_Wed__1_Jul_2026_00_04_01_+0900_OVSy2WWK_9aByzDJ--





^ permalink  raw  reply  [nested|flat] 125+ messages in thread

* [PATCH v38 01/11] Add a syntax to create Incrementally Maintainable Materialized Views
@ 2019-12-20 01:05 Yugo Nagata <[email protected]>
  0 siblings, 0 replies; 125+ messages in thread

From: Yugo Nagata @ 2019-12-20 01:05 UTC (permalink / raw)

Allow to create Incrementally Maintainable Materialized View (IMMV)
by using INCREMENTAL option in CREATE MATERIALIZED VIEW command
as follow:

     CREATE [INCREMANTAL] MATERIALIZED VIEW xxxxx AS SELECT ....;
---
 src/backend/parser/gram.y     | 32 +++++++++++++++++++++-----------
 src/include/nodes/primnodes.h |  1 +
 src/include/parser/kwlist.h   |  1 +
 3 files changed, 23 insertions(+), 11 deletions(-)

diff --git a/src/backend/parser/gram.y b/src/backend/parser/gram.y
index ff4e1388c55..9a353550c37 100644
--- a/src/backend/parser/gram.y
+++ b/src/backend/parser/gram.y
@@ -473,6 +473,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
 
 %type <range>	OptTempTableName
 %type <into>	into_clause create_as_target create_mv_target
+%type <boolean>	incremental
 
 %type <defelt>	createfunc_opt_item common_func_opt_item dostmt_opt_item
 %type <fun_param> func_arg func_arg_with_default table_func_column aggr_arg
@@ -776,7 +777,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
 	HANDLER HAVING HEADER_P HOLD HOUR_P
 
 	IDENTITY_P IF_P IGNORE_P ILIKE IMMEDIATE IMMUTABLE IMPLICIT_P IMPORT_P IN_P INCLUDE
-	INCLUDING INCREMENT INDENT INDEX INDEXES INHERIT INHERITS INITIALLY INLINE_P
+	INCLUDING INCREMENT INCREMENTAL INDENT INDEX INDEXES INHERIT INHERITS INITIALLY INLINE_P
 	INNER_P INOUT INPUT_P INSENSITIVE INSERT INSTEAD INT_P INTEGER
 	INTERSECT INTERVAL INTO INVOKER IS ISNULL ISOLATION
 
@@ -5039,32 +5040,34 @@ opt_with_data:
  *****************************************************************************/
 
 CreateMatViewStmt:
-		CREATE OptNoLog MATERIALIZED VIEW create_mv_target AS SelectStmt opt_with_data
+		CREATE OptNoLog incremental MATERIALIZED VIEW create_mv_target AS SelectStmt opt_with_data
 				{
 					CreateTableAsStmt *ctas = makeNode(CreateTableAsStmt);
 
-					ctas->query = $7;
-					ctas->into = $5;
+					ctas->query = $8;
+					ctas->into = $6;
 					ctas->objtype = OBJECT_MATVIEW;
 					ctas->is_select_into = false;
 					ctas->if_not_exists = false;
 					/* cram additional flags into the IntoClause */
-					$5->rel->relpersistence = $2;
-					$5->skipData = !($8);
+					$6->rel->relpersistence = $2;
+					$6->skipData = !($9);
+					$6->ivm = $3;
 					$$ = (Node *) ctas;
 				}
-		| CREATE OptNoLog MATERIALIZED VIEW IF_P NOT EXISTS create_mv_target AS SelectStmt opt_with_data
+		| CREATE OptNoLog incremental MATERIALIZED VIEW IF_P NOT EXISTS create_mv_target AS SelectStmt opt_with_data
 				{
 					CreateTableAsStmt *ctas = makeNode(CreateTableAsStmt);
 
-					ctas->query = $10;
-					ctas->into = $8;
+					ctas->query = $11;
+					ctas->into = $9;
 					ctas->objtype = OBJECT_MATVIEW;
 					ctas->is_select_into = false;
 					ctas->if_not_exists = true;
 					/* cram additional flags into the IntoClause */
-					$8->rel->relpersistence = $2;
-					$8->skipData = !($11);
+					$9->rel->relpersistence = $2;
+					$9->skipData = !($12);
+					$9->ivm = $3;
 					$$ = (Node *) ctas;
 				}
 		;
@@ -5081,9 +5084,14 @@ create_mv_target:
 					$$->tableSpaceName = $5;
 					$$->viewQuery = NULL;		/* filled at analysis time */
 					$$->skipData = false;		/* might get changed later */
+					$$->ivm = false;
 				}
 		;
 
+incremental:	INCREMENTAL				{ $$ = true; }
+				| /*EMPTY*/				{ $$ = false; }
+		;
+
 OptNoLog:	UNLOGGED					{ $$ = RELPERSISTENCE_UNLOGGED; }
 			| /*EMPTY*/					{ $$ = RELPERSISTENCE_PERMANENT; }
 		;
@@ -18948,6 +18956,7 @@ unreserved_keyword:
 			| INCLUDE
 			| INCLUDING
 			| INCREMENT
+			| INCREMENTAL
 			| INDENT
 			| INDEX
 			| INDEXES
@@ -19554,6 +19563,7 @@ bare_label_keyword:
 			| INCLUDE
 			| INCLUDING
 			| INCREMENT
+			| INCREMENTAL
 			| INDENT
 			| INDEX
 			| INDEXES
diff --git a/src/include/nodes/primnodes.h b/src/include/nodes/primnodes.h
index bb05aeebee4..da35b15cc27 100644
--- a/src/include/nodes/primnodes.h
+++ b/src/include/nodes/primnodes.h
@@ -170,6 +170,7 @@ typedef struct IntoClause
 	/* materialized view's SELECT query */
 	struct Query *viewQuery pg_node_attr(query_jumble_ignore);
 	bool		skipData;		/* true for WITH NO DATA */
+	bool		ivm;			/* true for WITH IVM */
 } IntoClause;
 
 
diff --git a/src/include/parser/kwlist.h b/src/include/parser/kwlist.h
index 51ead54f015..7849ee7f960 100644
--- a/src/include/parser/kwlist.h
+++ b/src/include/parser/kwlist.h
@@ -216,6 +216,7 @@ PG_KEYWORD("in", IN_P, RESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("include", INCLUDE, UNRESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("including", INCLUDING, UNRESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("increment", INCREMENT, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("incremental", INCREMENTAL, UNRESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("indent", INDENT, UNRESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("index", INDEX, UNRESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("indexes", INDEXES, UNRESERVED_KEYWORD, BARE_LABEL)
-- 
2.43.0


--Multipart=_Wed__1_Jul_2026_00_04_01_+0900_OVSy2WWK_9aByzDJ--





^ permalink  raw  reply  [nested|flat] 125+ messages in thread

* [PATCH v29 01/11] Add a syntax to create Incrementally Maintainable Materialized Views
@ 2019-12-20 01:05 Yugo Nagata <[email protected]>
  0 siblings, 0 replies; 125+ messages in thread

From: Yugo Nagata @ 2019-12-20 01:05 UTC (permalink / raw)

Allow to create Incrementally Maintainable Materialized View (IMMV)
by using INCREMENTAL option in CREATE MATERIALIZED VIEW command
as follow:

     CREATE [INCREMANTAL] MATERIALIZED VIEW xxxxx AS SELECT ....;
---
 src/backend/parser/gram.y     | 32 +++++++++++++++++++++-----------
 src/include/nodes/primnodes.h |  1 +
 src/include/parser/kwlist.h   |  1 +
 3 files changed, 23 insertions(+), 11 deletions(-)

diff --git a/src/backend/parser/gram.y b/src/backend/parser/gram.y
index 7d2032885e..33c647b0c7 100644
--- a/src/backend/parser/gram.y
+++ b/src/backend/parser/gram.y
@@ -465,6 +465,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
 
 %type <range>	OptTempTableName
 %type <into>	into_clause create_as_target create_mv_target
+%type <boolean>	incremental
 
 %type <defelt>	createfunc_opt_item common_func_opt_item dostmt_opt_item
 %type <fun_param> func_arg func_arg_with_default table_func_column aggr_arg
@@ -718,7 +719,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
 	HANDLER HAVING HEADER_P HOLD HOUR_P
 
 	IDENTITY_P IF_P ILIKE IMMEDIATE IMMUTABLE IMPLICIT_P IMPORT_P IN_P INCLUDE
-	INCLUDING INCREMENT INDENT INDEX INDEXES INHERIT INHERITS INITIALLY INLINE_P
+	INCLUDING INCREMENT INCREMENTAL INDENT INDEX INDEXES INHERIT INHERITS INITIALLY INLINE_P
 	INNER_P INOUT INPUT_P INSENSITIVE INSERT INSTEAD INT_P INTEGER
 	INTERSECT INTERVAL INTO INVOKER IS ISNULL ISOLATION
 
@@ -4652,32 +4653,34 @@ opt_with_data:
  *****************************************************************************/
 
 CreateMatViewStmt:
-		CREATE OptNoLog MATERIALIZED VIEW create_mv_target AS SelectStmt opt_with_data
+		CREATE OptNoLog incremental MATERIALIZED VIEW create_mv_target AS SelectStmt opt_with_data
 				{
 					CreateTableAsStmt *ctas = makeNode(CreateTableAsStmt);
 
-					ctas->query = $7;
-					ctas->into = $5;
+					ctas->query = $8;
+					ctas->into = $6;
 					ctas->objtype = OBJECT_MATVIEW;
 					ctas->is_select_into = false;
 					ctas->if_not_exists = false;
 					/* cram additional flags into the IntoClause */
-					$5->rel->relpersistence = $2;
-					$5->skipData = !($8);
+					$6->rel->relpersistence = $2;
+					$6->skipData = !($9);
+					$6->ivm = $3;
 					$$ = (Node *) ctas;
 				}
-		| CREATE OptNoLog MATERIALIZED VIEW IF_P NOT EXISTS create_mv_target AS SelectStmt opt_with_data
+		| CREATE OptNoLog incremental MATERIALIZED VIEW IF_P NOT EXISTS create_mv_target AS SelectStmt opt_with_data
 				{
 					CreateTableAsStmt *ctas = makeNode(CreateTableAsStmt);
 
-					ctas->query = $10;
-					ctas->into = $8;
+					ctas->query = $11;
+					ctas->into = $9;
 					ctas->objtype = OBJECT_MATVIEW;
 					ctas->is_select_into = false;
 					ctas->if_not_exists = true;
 					/* cram additional flags into the IntoClause */
-					$8->rel->relpersistence = $2;
-					$8->skipData = !($11);
+					$9->rel->relpersistence = $2;
+					$9->skipData = !($12);
+					$9->ivm = $3;
 					$$ = (Node *) ctas;
 				}
 		;
@@ -4694,9 +4697,14 @@ create_mv_target:
 					$$->tableSpaceName = $5;
 					$$->viewQuery = NULL;		/* filled at analysis time */
 					$$->skipData = false;		/* might get changed later */
+					$$->ivm = false;
 				}
 		;
 
+incremental:	INCREMENTAL				{ $$ = true; }
+				| /*EMPTY*/				{ $$ = false; }
+		;
+
 OptNoLog:	UNLOGGED					{ $$ = RELPERSISTENCE_UNLOGGED; }
 			| /*EMPTY*/					{ $$ = RELPERSISTENCE_PERMANENT; }
 		;
@@ -17141,6 +17149,7 @@ unreserved_keyword:
 			| INCLUDE
 			| INCLUDING
 			| INCREMENT
+			| INCREMENTAL
 			| INDENT
 			| INDEX
 			| INDEXES
@@ -17709,6 +17718,7 @@ bare_label_keyword:
 			| INCLUDE
 			| INCLUDING
 			| INCREMENT
+			| INCREMENTAL
 			| INDENT
 			| INDEX
 			| INDEXES
diff --git a/src/include/nodes/primnodes.h b/src/include/nodes/primnodes.h
index 60d72a876b..cecb968b36 100644
--- a/src/include/nodes/primnodes.h
+++ b/src/include/nodes/primnodes.h
@@ -146,6 +146,7 @@ typedef struct IntoClause
 	/* materialized view's SELECT query */
 	Node	   *viewQuery pg_node_attr(query_jumble_ignore);
 	bool		skipData;		/* true for WITH NO DATA */
+	bool		ivm;			/* true for WITH IVM */
 } IntoClause;
 
 
diff --git a/src/include/parser/kwlist.h b/src/include/parser/kwlist.h
index 5984dcfa4b..d60eb98d65 100644
--- a/src/include/parser/kwlist.h
+++ b/src/include/parser/kwlist.h
@@ -207,6 +207,7 @@ PG_KEYWORD("in", IN_P, RESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("include", INCLUDE, UNRESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("including", INCLUDING, UNRESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("increment", INCREMENT, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("incremental", INCREMENTAL, UNRESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("indent", INDENT, UNRESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("index", INDEX, UNRESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("indexes", INDEXES, UNRESERVED_KEYWORD, BARE_LABEL)
-- 
2.25.1


--Multipart=_Mon__28_Aug_2023_11_52_52_+0900_hj6L5h176QaSGtg7
Content-Type: text/x-diff;
 name="v29-0002-Add-relisivm-column-to-pg_class-system-catalog.patch"
Content-Disposition: attachment;
 filename="v29-0002-Add-relisivm-column-to-pg_class-system-catalog.patch"
Content-Transfer-Encoding: 7bit



^ permalink  raw  reply  [nested|flat] 125+ messages in thread

* [PATCH v28 01/11] Add a syntax to create Incrementally Maintainable Materialized Views
@ 2019-12-20 01:05 Yugo Nagata <[email protected]>
  0 siblings, 0 replies; 125+ messages in thread

From: Yugo Nagata @ 2019-12-20 01:05 UTC (permalink / raw)

Allow to create Incrementally Maintainable Materialized View (IMMV)
by using INCREMENTAL option in CREATE MATERIALIZED VIEW command
as follow:

     CREATE [INCREMANTAL] MATERIALIZED VIEW xxxxx AS SELECT ....;
---
 src/backend/parser/gram.y     | 32 +++++++++++++++++++++-----------
 src/include/nodes/primnodes.h |  1 +
 src/include/parser/kwlist.h   |  1 +
 3 files changed, 23 insertions(+), 11 deletions(-)

diff --git a/src/backend/parser/gram.y b/src/backend/parser/gram.y
index 39ab7eac0d..3534b37f81 100644
--- a/src/backend/parser/gram.y
+++ b/src/backend/parser/gram.y
@@ -465,6 +465,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
 
 %type <range>	OptTempTableName
 %type <into>	into_clause create_as_target create_mv_target
+%type <boolean>	incremental
 
 %type <defelt>	createfunc_opt_item common_func_opt_item dostmt_opt_item
 %type <fun_param> func_arg func_arg_with_default table_func_column aggr_arg
@@ -721,7 +722,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
 	HANDLER HAVING HEADER_P HOLD HOUR_P
 
 	IDENTITY_P IF_P ILIKE IMMEDIATE IMMUTABLE IMPLICIT_P IMPORT_P IN_P INCLUDE
-	INCLUDING INCREMENT INDENT INDEX INDEXES INHERIT INHERITS INITIALLY INLINE_P
+	INCLUDING INCREMENT INCREMENTAL INDENT INDEX INDEXES INHERIT INHERITS INITIALLY INLINE_P
 	INNER_P INOUT INPUT_P INSENSITIVE INSERT INSTEAD INT_P INTEGER
 	INTERSECT INTERVAL INTO INVOKER IS ISNULL ISOLATION
 
@@ -4637,32 +4638,34 @@ opt_with_data:
  *****************************************************************************/
 
 CreateMatViewStmt:
-		CREATE OptNoLog MATERIALIZED VIEW create_mv_target AS SelectStmt opt_with_data
+		CREATE OptNoLog incremental MATERIALIZED VIEW create_mv_target AS SelectStmt opt_with_data
 				{
 					CreateTableAsStmt *ctas = makeNode(CreateTableAsStmt);
 
-					ctas->query = $7;
-					ctas->into = $5;
+					ctas->query = $8;
+					ctas->into = $6;
 					ctas->objtype = OBJECT_MATVIEW;
 					ctas->is_select_into = false;
 					ctas->if_not_exists = false;
 					/* cram additional flags into the IntoClause */
-					$5->rel->relpersistence = $2;
-					$5->skipData = !($8);
+					$6->rel->relpersistence = $2;
+					$6->skipData = !($9);
+					$6->ivm = $3;
 					$$ = (Node *) ctas;
 				}
-		| CREATE OptNoLog MATERIALIZED VIEW IF_P NOT EXISTS create_mv_target AS SelectStmt opt_with_data
+		| CREATE OptNoLog incremental MATERIALIZED VIEW IF_P NOT EXISTS create_mv_target AS SelectStmt opt_with_data
 				{
 					CreateTableAsStmt *ctas = makeNode(CreateTableAsStmt);
 
-					ctas->query = $10;
-					ctas->into = $8;
+					ctas->query = $11;
+					ctas->into = $9;
 					ctas->objtype = OBJECT_MATVIEW;
 					ctas->is_select_into = false;
 					ctas->if_not_exists = true;
 					/* cram additional flags into the IntoClause */
-					$8->rel->relpersistence = $2;
-					$8->skipData = !($11);
+					$9->rel->relpersistence = $2;
+					$9->skipData = !($12);
+					$9->ivm = $3;
 					$$ = (Node *) ctas;
 				}
 		;
@@ -4679,9 +4682,14 @@ create_mv_target:
 					$$->tableSpaceName = $5;
 					$$->viewQuery = NULL;		/* filled at analysis time */
 					$$->skipData = false;		/* might get changed later */
+					$$->ivm = false;
 				}
 		;
 
+incremental:	INCREMENTAL				{ $$ = true; }
+				| /*EMPTY*/				{ $$ = false; }
+		;
+
 OptNoLog:	UNLOGGED					{ $$ = RELPERSISTENCE_UNLOGGED; }
 			| /*EMPTY*/					{ $$ = RELPERSISTENCE_PERMANENT; }
 		;
@@ -17055,6 +17063,7 @@ unreserved_keyword:
 			| INCLUDE
 			| INCLUDING
 			| INCREMENT
+			| INCREMENTAL
 			| INDENT
 			| INDEX
 			| INDEXES
@@ -17621,6 +17630,7 @@ bare_label_keyword:
 			| INCLUDE
 			| INCLUDING
 			| INCREMENT
+			| INCREMENTAL
 			| INDENT
 			| INDEX
 			| INDEXES
diff --git a/src/include/nodes/primnodes.h b/src/include/nodes/primnodes.h
index 792a743f72..e87a105707 100644
--- a/src/include/nodes/primnodes.h
+++ b/src/include/nodes/primnodes.h
@@ -146,6 +146,7 @@ typedef struct IntoClause
 	/* materialized view's SELECT query */
 	Node	   *viewQuery pg_node_attr(query_jumble_ignore);
 	bool		skipData;		/* true for WITH NO DATA */
+	bool		ivm;			/* true for WITH IVM */
 } IntoClause;
 
 
diff --git a/src/include/parser/kwlist.h b/src/include/parser/kwlist.h
index f5b2e61ca5..a3fbfedbf4 100644
--- a/src/include/parser/kwlist.h
+++ b/src/include/parser/kwlist.h
@@ -207,6 +207,7 @@ PG_KEYWORD("in", IN_P, RESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("include", INCLUDE, UNRESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("including", INCLUDING, UNRESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("increment", INCREMENT, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("incremental", INCREMENTAL, UNRESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("indent", INDENT, UNRESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("index", INDEX, UNRESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("indexes", INDEXES, UNRESERVED_KEYWORD, BARE_LABEL)
-- 
2.25.1


--Multipart=_Thu__1_Jun_2023_23_59_09_+0900_/G5+8nG46.f1T42K
Content-Type: text/x-diff;
 name="v28-0002-Add-relisivm-column-to-pg_class-system-catalog.patch"
Content-Disposition: attachment;
 filename="v28-0002-Add-relisivm-column-to-pg_class-system-catalog.patch"
Content-Transfer-Encoding: 7bit



^ permalink  raw  reply  [nested|flat] 125+ messages in thread

* [PATCH v38 01/11] Add a syntax to create Incrementally Maintainable Materialized Views
@ 2019-12-20 01:05 Yugo Nagata <[email protected]>
  0 siblings, 0 replies; 125+ messages in thread

From: Yugo Nagata @ 2019-12-20 01:05 UTC (permalink / raw)

Allow to create Incrementally Maintainable Materialized View (IMMV)
by using INCREMENTAL option in CREATE MATERIALIZED VIEW command
as follow:

     CREATE [INCREMANTAL] MATERIALIZED VIEW xxxxx AS SELECT ....;
---
 src/backend/parser/gram.y     | 32 +++++++++++++++++++++-----------
 src/include/nodes/primnodes.h |  1 +
 src/include/parser/kwlist.h   |  1 +
 3 files changed, 23 insertions(+), 11 deletions(-)

diff --git a/src/backend/parser/gram.y b/src/backend/parser/gram.y
index ff4e1388c55..9a353550c37 100644
--- a/src/backend/parser/gram.y
+++ b/src/backend/parser/gram.y
@@ -473,6 +473,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
 
 %type <range>	OptTempTableName
 %type <into>	into_clause create_as_target create_mv_target
+%type <boolean>	incremental
 
 %type <defelt>	createfunc_opt_item common_func_opt_item dostmt_opt_item
 %type <fun_param> func_arg func_arg_with_default table_func_column aggr_arg
@@ -776,7 +777,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
 	HANDLER HAVING HEADER_P HOLD HOUR_P
 
 	IDENTITY_P IF_P IGNORE_P ILIKE IMMEDIATE IMMUTABLE IMPLICIT_P IMPORT_P IN_P INCLUDE
-	INCLUDING INCREMENT INDENT INDEX INDEXES INHERIT INHERITS INITIALLY INLINE_P
+	INCLUDING INCREMENT INCREMENTAL INDENT INDEX INDEXES INHERIT INHERITS INITIALLY INLINE_P
 	INNER_P INOUT INPUT_P INSENSITIVE INSERT INSTEAD INT_P INTEGER
 	INTERSECT INTERVAL INTO INVOKER IS ISNULL ISOLATION
 
@@ -5039,32 +5040,34 @@ opt_with_data:
  *****************************************************************************/
 
 CreateMatViewStmt:
-		CREATE OptNoLog MATERIALIZED VIEW create_mv_target AS SelectStmt opt_with_data
+		CREATE OptNoLog incremental MATERIALIZED VIEW create_mv_target AS SelectStmt opt_with_data
 				{
 					CreateTableAsStmt *ctas = makeNode(CreateTableAsStmt);
 
-					ctas->query = $7;
-					ctas->into = $5;
+					ctas->query = $8;
+					ctas->into = $6;
 					ctas->objtype = OBJECT_MATVIEW;
 					ctas->is_select_into = false;
 					ctas->if_not_exists = false;
 					/* cram additional flags into the IntoClause */
-					$5->rel->relpersistence = $2;
-					$5->skipData = !($8);
+					$6->rel->relpersistence = $2;
+					$6->skipData = !($9);
+					$6->ivm = $3;
 					$$ = (Node *) ctas;
 				}
-		| CREATE OptNoLog MATERIALIZED VIEW IF_P NOT EXISTS create_mv_target AS SelectStmt opt_with_data
+		| CREATE OptNoLog incremental MATERIALIZED VIEW IF_P NOT EXISTS create_mv_target AS SelectStmt opt_with_data
 				{
 					CreateTableAsStmt *ctas = makeNode(CreateTableAsStmt);
 
-					ctas->query = $10;
-					ctas->into = $8;
+					ctas->query = $11;
+					ctas->into = $9;
 					ctas->objtype = OBJECT_MATVIEW;
 					ctas->is_select_into = false;
 					ctas->if_not_exists = true;
 					/* cram additional flags into the IntoClause */
-					$8->rel->relpersistence = $2;
-					$8->skipData = !($11);
+					$9->rel->relpersistence = $2;
+					$9->skipData = !($12);
+					$9->ivm = $3;
 					$$ = (Node *) ctas;
 				}
 		;
@@ -5081,9 +5084,14 @@ create_mv_target:
 					$$->tableSpaceName = $5;
 					$$->viewQuery = NULL;		/* filled at analysis time */
 					$$->skipData = false;		/* might get changed later */
+					$$->ivm = false;
 				}
 		;
 
+incremental:	INCREMENTAL				{ $$ = true; }
+				| /*EMPTY*/				{ $$ = false; }
+		;
+
 OptNoLog:	UNLOGGED					{ $$ = RELPERSISTENCE_UNLOGGED; }
 			| /*EMPTY*/					{ $$ = RELPERSISTENCE_PERMANENT; }
 		;
@@ -18948,6 +18956,7 @@ unreserved_keyword:
 			| INCLUDE
 			| INCLUDING
 			| INCREMENT
+			| INCREMENTAL
 			| INDENT
 			| INDEX
 			| INDEXES
@@ -19554,6 +19563,7 @@ bare_label_keyword:
 			| INCLUDE
 			| INCLUDING
 			| INCREMENT
+			| INCREMENTAL
 			| INDENT
 			| INDEX
 			| INDEXES
diff --git a/src/include/nodes/primnodes.h b/src/include/nodes/primnodes.h
index bb05aeebee4..da35b15cc27 100644
--- a/src/include/nodes/primnodes.h
+++ b/src/include/nodes/primnodes.h
@@ -170,6 +170,7 @@ typedef struct IntoClause
 	/* materialized view's SELECT query */
 	struct Query *viewQuery pg_node_attr(query_jumble_ignore);
 	bool		skipData;		/* true for WITH NO DATA */
+	bool		ivm;			/* true for WITH IVM */
 } IntoClause;
 
 
diff --git a/src/include/parser/kwlist.h b/src/include/parser/kwlist.h
index 51ead54f015..7849ee7f960 100644
--- a/src/include/parser/kwlist.h
+++ b/src/include/parser/kwlist.h
@@ -216,6 +216,7 @@ PG_KEYWORD("in", IN_P, RESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("include", INCLUDE, UNRESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("including", INCLUDING, UNRESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("increment", INCREMENT, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("incremental", INCREMENTAL, UNRESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("indent", INDENT, UNRESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("index", INDEX, UNRESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("indexes", INDEXES, UNRESERVED_KEYWORD, BARE_LABEL)
-- 
2.43.0


--Multipart=_Wed__1_Jul_2026_00_04_01_+0900_OVSy2WWK_9aByzDJ--





^ permalink  raw  reply  [nested|flat] 125+ messages in thread

* [PATCH v39 1/8] Add a syntax to create Incrementally Maintainable Materialized Views
@ 2019-12-20 01:05 Yugo Nagata <[email protected]>
  0 siblings, 0 replies; 125+ messages in thread

From: Yugo Nagata @ 2019-12-20 01:05 UTC (permalink / raw)

Allow to create Incrementally Maintainable Materialized View (IMMV)
by using INCREMENTAL option in CREATE MATERIALIZED VIEW command
as follow:

     CREATE [INCREMANTAL] MATERIALIZED VIEW xxxxx AS SELECT ....;
---
 src/backend/parser/gram.y     | 32 +++++++++++++++++++++-----------
 src/include/nodes/primnodes.h |  1 +
 src/include/parser/kwlist.h   |  1 +
 3 files changed, 23 insertions(+), 11 deletions(-)

diff --git a/src/backend/parser/gram.y b/src/backend/parser/gram.y
index ff4e1388c55..9a353550c37 100644
--- a/src/backend/parser/gram.y
+++ b/src/backend/parser/gram.y
@@ -473,6 +473,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
 
 %type <range>	OptTempTableName
 %type <into>	into_clause create_as_target create_mv_target
+%type <boolean>	incremental
 
 %type <defelt>	createfunc_opt_item common_func_opt_item dostmt_opt_item
 %type <fun_param> func_arg func_arg_with_default table_func_column aggr_arg
@@ -776,7 +777,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
 	HANDLER HAVING HEADER_P HOLD HOUR_P
 
 	IDENTITY_P IF_P IGNORE_P ILIKE IMMEDIATE IMMUTABLE IMPLICIT_P IMPORT_P IN_P INCLUDE
-	INCLUDING INCREMENT INDENT INDEX INDEXES INHERIT INHERITS INITIALLY INLINE_P
+	INCLUDING INCREMENT INCREMENTAL INDENT INDEX INDEXES INHERIT INHERITS INITIALLY INLINE_P
 	INNER_P INOUT INPUT_P INSENSITIVE INSERT INSTEAD INT_P INTEGER
 	INTERSECT INTERVAL INTO INVOKER IS ISNULL ISOLATION
 
@@ -5039,32 +5040,34 @@ opt_with_data:
  *****************************************************************************/
 
 CreateMatViewStmt:
-		CREATE OptNoLog MATERIALIZED VIEW create_mv_target AS SelectStmt opt_with_data
+		CREATE OptNoLog incremental MATERIALIZED VIEW create_mv_target AS SelectStmt opt_with_data
 				{
 					CreateTableAsStmt *ctas = makeNode(CreateTableAsStmt);
 
-					ctas->query = $7;
-					ctas->into = $5;
+					ctas->query = $8;
+					ctas->into = $6;
 					ctas->objtype = OBJECT_MATVIEW;
 					ctas->is_select_into = false;
 					ctas->if_not_exists = false;
 					/* cram additional flags into the IntoClause */
-					$5->rel->relpersistence = $2;
-					$5->skipData = !($8);
+					$6->rel->relpersistence = $2;
+					$6->skipData = !($9);
+					$6->ivm = $3;
 					$$ = (Node *) ctas;
 				}
-		| CREATE OptNoLog MATERIALIZED VIEW IF_P NOT EXISTS create_mv_target AS SelectStmt opt_with_data
+		| CREATE OptNoLog incremental MATERIALIZED VIEW IF_P NOT EXISTS create_mv_target AS SelectStmt opt_with_data
 				{
 					CreateTableAsStmt *ctas = makeNode(CreateTableAsStmt);
 
-					ctas->query = $10;
-					ctas->into = $8;
+					ctas->query = $11;
+					ctas->into = $9;
 					ctas->objtype = OBJECT_MATVIEW;
 					ctas->is_select_into = false;
 					ctas->if_not_exists = true;
 					/* cram additional flags into the IntoClause */
-					$8->rel->relpersistence = $2;
-					$8->skipData = !($11);
+					$9->rel->relpersistence = $2;
+					$9->skipData = !($12);
+					$9->ivm = $3;
 					$$ = (Node *) ctas;
 				}
 		;
@@ -5081,9 +5084,14 @@ create_mv_target:
 					$$->tableSpaceName = $5;
 					$$->viewQuery = NULL;		/* filled at analysis time */
 					$$->skipData = false;		/* might get changed later */
+					$$->ivm = false;
 				}
 		;
 
+incremental:	INCREMENTAL				{ $$ = true; }
+				| /*EMPTY*/				{ $$ = false; }
+		;
+
 OptNoLog:	UNLOGGED					{ $$ = RELPERSISTENCE_UNLOGGED; }
 			| /*EMPTY*/					{ $$ = RELPERSISTENCE_PERMANENT; }
 		;
@@ -18948,6 +18956,7 @@ unreserved_keyword:
 			| INCLUDE
 			| INCLUDING
 			| INCREMENT
+			| INCREMENTAL
 			| INDENT
 			| INDEX
 			| INDEXES
@@ -19554,6 +19563,7 @@ bare_label_keyword:
 			| INCLUDE
 			| INCLUDING
 			| INCREMENT
+			| INCREMENTAL
 			| INDENT
 			| INDEX
 			| INDEXES
diff --git a/src/include/nodes/primnodes.h b/src/include/nodes/primnodes.h
index bb05aeebee4..da35b15cc27 100644
--- a/src/include/nodes/primnodes.h
+++ b/src/include/nodes/primnodes.h
@@ -170,6 +170,7 @@ typedef struct IntoClause
 	/* materialized view's SELECT query */
 	struct Query *viewQuery pg_node_attr(query_jumble_ignore);
 	bool		skipData;		/* true for WITH NO DATA */
+	bool		ivm;			/* true for WITH IVM */
 } IntoClause;
 
 
diff --git a/src/include/parser/kwlist.h b/src/include/parser/kwlist.h
index 51ead54f015..7849ee7f960 100644
--- a/src/include/parser/kwlist.h
+++ b/src/include/parser/kwlist.h
@@ -216,6 +216,7 @@ PG_KEYWORD("in", IN_P, RESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("include", INCLUDE, UNRESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("including", INCLUDING, UNRESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("increment", INCREMENT, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("incremental", INCREMENTAL, UNRESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("indent", INDENT, UNRESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("index", INDEX, UNRESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("indexes", INDEXES, UNRESERVED_KEYWORD, BARE_LABEL)
-- 
2.43.0


--Multipart=_Fri__3_Jul_2026_19_11_16_+0900_CskxSIu_iGcDMEyM--






^ permalink  raw  reply  [nested|flat] 125+ messages in thread

* [PATCH v37 01/11] Add a syntax to create Incrementally Maintainable Materialized Views
@ 2019-12-20 01:05 Yugo Nagata <[email protected]>
  0 siblings, 0 replies; 125+ messages in thread

From: Yugo Nagata @ 2019-12-20 01:05 UTC (permalink / raw)

Allow to create Incrementally Maintainable Materialized View (IMMV)
by using INCREMENTAL option in CREATE MATERIALIZED VIEW command
as follow:

     CREATE [INCREMANTAL] MATERIALIZED VIEW xxxxx AS SELECT ....;
---
 src/backend/parser/gram.y     | 32 +++++++++++++++++++++-----------
 src/include/nodes/primnodes.h |  1 +
 src/include/parser/kwlist.h   |  1 +
 3 files changed, 23 insertions(+), 11 deletions(-)

diff --git a/src/backend/parser/gram.y b/src/backend/parser/gram.y
index ff4e1388c55..9a353550c37 100644
--- a/src/backend/parser/gram.y
+++ b/src/backend/parser/gram.y
@@ -473,6 +473,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
 
 %type <range>	OptTempTableName
 %type <into>	into_clause create_as_target create_mv_target
+%type <boolean>	incremental
 
 %type <defelt>	createfunc_opt_item common_func_opt_item dostmt_opt_item
 %type <fun_param> func_arg func_arg_with_default table_func_column aggr_arg
@@ -776,7 +777,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
 	HANDLER HAVING HEADER_P HOLD HOUR_P
 
 	IDENTITY_P IF_P IGNORE_P ILIKE IMMEDIATE IMMUTABLE IMPLICIT_P IMPORT_P IN_P INCLUDE
-	INCLUDING INCREMENT INDENT INDEX INDEXES INHERIT INHERITS INITIALLY INLINE_P
+	INCLUDING INCREMENT INCREMENTAL INDENT INDEX INDEXES INHERIT INHERITS INITIALLY INLINE_P
 	INNER_P INOUT INPUT_P INSENSITIVE INSERT INSTEAD INT_P INTEGER
 	INTERSECT INTERVAL INTO INVOKER IS ISNULL ISOLATION
 
@@ -5039,32 +5040,34 @@ opt_with_data:
  *****************************************************************************/
 
 CreateMatViewStmt:
-		CREATE OptNoLog MATERIALIZED VIEW create_mv_target AS SelectStmt opt_with_data
+		CREATE OptNoLog incremental MATERIALIZED VIEW create_mv_target AS SelectStmt opt_with_data
 				{
 					CreateTableAsStmt *ctas = makeNode(CreateTableAsStmt);
 
-					ctas->query = $7;
-					ctas->into = $5;
+					ctas->query = $8;
+					ctas->into = $6;
 					ctas->objtype = OBJECT_MATVIEW;
 					ctas->is_select_into = false;
 					ctas->if_not_exists = false;
 					/* cram additional flags into the IntoClause */
-					$5->rel->relpersistence = $2;
-					$5->skipData = !($8);
+					$6->rel->relpersistence = $2;
+					$6->skipData = !($9);
+					$6->ivm = $3;
 					$$ = (Node *) ctas;
 				}
-		| CREATE OptNoLog MATERIALIZED VIEW IF_P NOT EXISTS create_mv_target AS SelectStmt opt_with_data
+		| CREATE OptNoLog incremental MATERIALIZED VIEW IF_P NOT EXISTS create_mv_target AS SelectStmt opt_with_data
 				{
 					CreateTableAsStmt *ctas = makeNode(CreateTableAsStmt);
 
-					ctas->query = $10;
-					ctas->into = $8;
+					ctas->query = $11;
+					ctas->into = $9;
 					ctas->objtype = OBJECT_MATVIEW;
 					ctas->is_select_into = false;
 					ctas->if_not_exists = true;
 					/* cram additional flags into the IntoClause */
-					$8->rel->relpersistence = $2;
-					$8->skipData = !($11);
+					$9->rel->relpersistence = $2;
+					$9->skipData = !($12);
+					$9->ivm = $3;
 					$$ = (Node *) ctas;
 				}
 		;
@@ -5081,9 +5084,14 @@ create_mv_target:
 					$$->tableSpaceName = $5;
 					$$->viewQuery = NULL;		/* filled at analysis time */
 					$$->skipData = false;		/* might get changed later */
+					$$->ivm = false;
 				}
 		;
 
+incremental:	INCREMENTAL				{ $$ = true; }
+				| /*EMPTY*/				{ $$ = false; }
+		;
+
 OptNoLog:	UNLOGGED					{ $$ = RELPERSISTENCE_UNLOGGED; }
 			| /*EMPTY*/					{ $$ = RELPERSISTENCE_PERMANENT; }
 		;
@@ -18948,6 +18956,7 @@ unreserved_keyword:
 			| INCLUDE
 			| INCLUDING
 			| INCREMENT
+			| INCREMENTAL
 			| INDENT
 			| INDEX
 			| INDEXES
@@ -19554,6 +19563,7 @@ bare_label_keyword:
 			| INCLUDE
 			| INCLUDING
 			| INCREMENT
+			| INCREMENTAL
 			| INDENT
 			| INDEX
 			| INDEXES
diff --git a/src/include/nodes/primnodes.h b/src/include/nodes/primnodes.h
index 7977ee24783..213adeec2e2 100644
--- a/src/include/nodes/primnodes.h
+++ b/src/include/nodes/primnodes.h
@@ -170,6 +170,7 @@ typedef struct IntoClause
 	/* materialized view's SELECT query */
 	struct Query *viewQuery pg_node_attr(query_jumble_ignore);
 	bool		skipData;		/* true for WITH NO DATA */
+	bool		ivm;			/* true for WITH IVM */
 } IntoClause;
 
 
diff --git a/src/include/parser/kwlist.h b/src/include/parser/kwlist.h
index 51ead54f015..7849ee7f960 100644
--- a/src/include/parser/kwlist.h
+++ b/src/include/parser/kwlist.h
@@ -216,6 +216,7 @@ PG_KEYWORD("in", IN_P, RESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("include", INCLUDE, UNRESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("including", INCLUDING, UNRESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("increment", INCREMENT, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("incremental", INCREMENTAL, UNRESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("indent", INDENT, UNRESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("index", INDEX, UNRESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("indexes", INDEXES, UNRESERVED_KEYWORD, BARE_LABEL)
-- 
2.43.0


--Multipart=_Fri__29_May_2026_23_14_17_+0900_Te0o73X2VqYK57Gd--





^ permalink  raw  reply  [nested|flat] 125+ messages in thread

* [PATCH v25 01/15] Add a syntax to create Incrementally Maintainable Materialized Views
@ 2019-12-20 01:05 Yugo Nagata <[email protected]>
  0 siblings, 0 replies; 125+ messages in thread

From: Yugo Nagata @ 2019-12-20 01:05 UTC (permalink / raw)

Allow to create Incrementally Maintainable Materialized View (IMMV)
by using INCREMENTAL option in CREATE MATERIALIZED VIEW command
as follow:

     CREATE [INCREMANTAL] MATERIALIZED VIEW xxxxx AS SELECT ....;
---
 src/backend/nodes/copyfuncs.c  |  1 +
 src/backend/nodes/equalfuncs.c |  1 +
 src/backend/nodes/outfuncs.c   |  1 +
 src/backend/nodes/readfuncs.c  |  1 +
 src/backend/parser/gram.y      | 32 +++++++++++++++++++++-----------
 src/include/nodes/primnodes.h  |  1 +
 src/include/parser/kwlist.h    |  1 +
 7 files changed, 27 insertions(+), 11 deletions(-)

diff --git a/src/backend/nodes/copyfuncs.c b/src/backend/nodes/copyfuncs.c
index 90b5da51c9..80268ac059 100644
--- a/src/backend/nodes/copyfuncs.c
+++ b/src/backend/nodes/copyfuncs.c
@@ -1426,6 +1426,7 @@ _copyIntoClause(const IntoClause *from)
 	COPY_STRING_FIELD(tableSpaceName);
 	COPY_NODE_FIELD(viewQuery);
 	COPY_SCALAR_FIELD(skipData);
+	COPY_SCALAR_FIELD(ivm);
 
 	return newnode;
 }
diff --git a/src/backend/nodes/equalfuncs.c b/src/backend/nodes/equalfuncs.c
index 06345da3ba..572560b4a2 100644
--- a/src/backend/nodes/equalfuncs.c
+++ b/src/backend/nodes/equalfuncs.c
@@ -155,6 +155,7 @@ _equalIntoClause(const IntoClause *a, const IntoClause *b)
 	COMPARE_STRING_FIELD(tableSpaceName);
 	COMPARE_NODE_FIELD(viewQuery);
 	COMPARE_SCALAR_FIELD(skipData);
+	COMPARE_SCALAR_FIELD(ivm);
 
 	return true;
 }
diff --git a/src/backend/nodes/outfuncs.c b/src/backend/nodes/outfuncs.c
index 2b0236937a..8ec4059cc0 100644
--- a/src/backend/nodes/outfuncs.c
+++ b/src/backend/nodes/outfuncs.c
@@ -1119,6 +1119,7 @@ _outIntoClause(StringInfo str, const IntoClause *node)
 	WRITE_STRING_FIELD(tableSpaceName);
 	WRITE_NODE_FIELD(viewQuery);
 	WRITE_BOOL_FIELD(skipData);
+	WRITE_BOOL_FIELD(ivm);
 }
 
 static void
diff --git a/src/backend/nodes/readfuncs.c b/src/backend/nodes/readfuncs.c
index 3f68f7c18d..cc6dcb7220 100644
--- a/src/backend/nodes/readfuncs.c
+++ b/src/backend/nodes/readfuncs.c
@@ -563,6 +563,7 @@ _readIntoClause(void)
 	READ_STRING_FIELD(tableSpaceName);
 	READ_NODE_FIELD(viewQuery);
 	READ_BOOL_FIELD(skipData);
+	READ_BOOL_FIELD(ivm);
 
 	READ_DONE();
 }
diff --git a/src/backend/parser/gram.y b/src/backend/parser/gram.y
index b5966712ce..21beb087c2 100644
--- a/src/backend/parser/gram.y
+++ b/src/backend/parser/gram.y
@@ -459,6 +459,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
 
 %type <range>	OptTempTableName
 %type <into>	into_clause create_as_target create_mv_target
+%type <boolean>	incremental
 
 %type <defelt>	createfunc_opt_item common_func_opt_item dostmt_opt_item
 %type <fun_param> func_arg func_arg_with_default table_func_column aggr_arg
@@ -692,7 +693,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
 	HANDLER HAVING HEADER_P HOLD HOUR_P
 
 	IDENTITY_P IF_P ILIKE IMMEDIATE IMMUTABLE IMPLICIT_P IMPORT_P IN_P INCLUDE
-	INCLUDING INCREMENT INDEX INDEXES INHERIT INHERITS INITIALLY INLINE_P
+	INCLUDING INCREMENT INCREMENTAL INDEX INDEXES INHERIT INHERITS INITIALLY INLINE_P
 	INNER_P INOUT INPUT_P INSENSITIVE INSERT INSTEAD INT_P INTEGER
 	INTERSECT INTERVAL INTO INVOKER IS ISNULL ISOLATION
 
@@ -4359,30 +4360,32 @@ opt_with_data:
  *****************************************************************************/
 
 CreateMatViewStmt:
-		CREATE OptNoLog MATERIALIZED VIEW create_mv_target AS SelectStmt opt_with_data
+		CREATE OptNoLog incremental MATERIALIZED VIEW create_mv_target AS SelectStmt opt_with_data
 				{
 					CreateTableAsStmt *ctas = makeNode(CreateTableAsStmt);
-					ctas->query = $7;
-					ctas->into = $5;
+					ctas->query = $8;
+					ctas->into = $6;
 					ctas->objtype = OBJECT_MATVIEW;
 					ctas->is_select_into = false;
 					ctas->if_not_exists = false;
 					/* cram additional flags into the IntoClause */
-					$5->rel->relpersistence = $2;
-					$5->skipData = !($8);
+					$6->rel->relpersistence = $2;
+					$6->skipData = !($9);
+					$6->ivm = $3;
 					$$ = (Node *) ctas;
 				}
-		| CREATE OptNoLog MATERIALIZED VIEW IF_P NOT EXISTS create_mv_target AS SelectStmt opt_with_data
+		| CREATE OptNoLog incremental MATERIALIZED VIEW IF_P NOT EXISTS create_mv_target AS SelectStmt opt_with_data
 				{
 					CreateTableAsStmt *ctas = makeNode(CreateTableAsStmt);
-					ctas->query = $10;
-					ctas->into = $8;
+					ctas->query = $11;
+					ctas->into = $9;
 					ctas->objtype = OBJECT_MATVIEW;
 					ctas->is_select_into = false;
 					ctas->if_not_exists = true;
 					/* cram additional flags into the IntoClause */
-					$8->rel->relpersistence = $2;
-					$8->skipData = !($11);
+					$9->rel->relpersistence = $2;
+					$9->skipData = !($12);
+					$9->ivm = $3;
 					$$ = (Node *) ctas;
 				}
 		;
@@ -4399,9 +4402,14 @@ create_mv_target:
 					$$->tableSpaceName = $5;
 					$$->viewQuery = NULL;		/* filled at analysis time */
 					$$->skipData = false;		/* might get changed later */
+					$$->ivm = false;
 				}
 		;
 
+incremental:	INCREMENTAL				{ $$ = true; }
+				| /*EMPTY*/				{ $$ = false; }
+		;
+
 OptNoLog:	UNLOGGED					{ $$ = RELPERSISTENCE_UNLOGGED; }
 			| /*EMPTY*/					{ $$ = RELPERSISTENCE_PERMANENT; }
 		;
@@ -15752,6 +15760,7 @@ unreserved_keyword:
 			| INCLUDE
 			| INCLUDING
 			| INCREMENT
+			| INCREMENTAL
 			| INDEX
 			| INDEXES
 			| INHERIT
@@ -16304,6 +16313,7 @@ bare_label_keyword:
 			| INCLUDE
 			| INCLUDING
 			| INCREMENT
+			| INCREMENTAL
 			| INDEX
 			| INDEXES
 			| INHERIT
diff --git a/src/include/nodes/primnodes.h b/src/include/nodes/primnodes.h
index dab5c4ff5d..bc1bcbda13 100644
--- a/src/include/nodes/primnodes.h
+++ b/src/include/nodes/primnodes.h
@@ -117,6 +117,7 @@ typedef struct IntoClause
 	char	   *tableSpaceName; /* table space to use, or NULL */
 	Node	   *viewQuery;		/* materialized view's SELECT query */
 	bool		skipData;		/* true for WITH NO DATA */
+	bool		ivm;			/* true for WITH IVM */
 } IntoClause;
 
 
diff --git a/src/include/parser/kwlist.h b/src/include/parser/kwlist.h
index bcef7eed2f..a43af4a597 100644
--- a/src/include/parser/kwlist.h
+++ b/src/include/parser/kwlist.h
@@ -205,6 +205,7 @@ PG_KEYWORD("in", IN_P, RESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("include", INCLUDE, UNRESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("including", INCLUDING, UNRESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("increment", INCREMENT, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("incremental", INCREMENTAL, UNRESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("index", INDEX, UNRESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("indexes", INDEXES, UNRESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("inherit", INHERIT, UNRESERVED_KEYWORD, BARE_LABEL)
-- 
2.17.1


--Multipart=_Fri__4_Feb_2022_01_48_06_+0900_N8BNZpfOR27sWrgY--





^ permalink  raw  reply  [nested|flat] 125+ messages in thread

* [PATCH v30 01/11] Add a syntax to create Incrementally Maintainable Materialized Views
@ 2019-12-20 01:05 Yugo Nagata <[email protected]>
  0 siblings, 0 replies; 125+ messages in thread

From: Yugo Nagata @ 2019-12-20 01:05 UTC (permalink / raw)

Allow to create Incrementally Maintainable Materialized View (IMMV)
by using INCREMENTAL option in CREATE MATERIALIZED VIEW command
as follow:

     CREATE [INCREMANTAL] MATERIALIZED VIEW xxxxx AS SELECT ....;
---
 src/backend/parser/gram.y     | 32 +++++++++++++++++++++-----------
 src/include/nodes/primnodes.h |  1 +
 src/include/parser/kwlist.h   |  1 +
 3 files changed, 23 insertions(+), 11 deletions(-)

diff --git a/src/backend/parser/gram.y b/src/backend/parser/gram.y
index 130f7fc7c3..7ba972ae5d 100644
--- a/src/backend/parser/gram.y
+++ b/src/backend/parser/gram.y
@@ -466,6 +466,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
 
 %type <range>	OptTempTableName
 %type <into>	into_clause create_as_target create_mv_target
+%type <boolean>	incremental
 
 %type <defelt>	createfunc_opt_item common_func_opt_item dostmt_opt_item
 %type <fun_param> func_arg func_arg_with_default table_func_column aggr_arg
@@ -719,7 +720,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
 	HANDLER HAVING HEADER_P HOLD HOUR_P
 
 	IDENTITY_P IF_P ILIKE IMMEDIATE IMMUTABLE IMPLICIT_P IMPORT_P IN_P INCLUDE
-	INCLUDING INCREMENT INDENT INDEX INDEXES INHERIT INHERITS INITIALLY INLINE_P
+	INCLUDING INCREMENT INCREMENTAL INDENT INDEX INDEXES INHERIT INHERITS INITIALLY INLINE_P
 	INNER_P INOUT INPUT_P INSENSITIVE INSERT INSTEAD INT_P INTEGER
 	INTERSECT INTERVAL INTO INVOKER IS ISNULL ISOLATION
 
@@ -4696,32 +4697,34 @@ opt_with_data:
  *****************************************************************************/
 
 CreateMatViewStmt:
-		CREATE OptNoLog MATERIALIZED VIEW create_mv_target AS SelectStmt opt_with_data
+		CREATE OptNoLog incremental MATERIALIZED VIEW create_mv_target AS SelectStmt opt_with_data
 				{
 					CreateTableAsStmt *ctas = makeNode(CreateTableAsStmt);
 
-					ctas->query = $7;
-					ctas->into = $5;
+					ctas->query = $8;
+					ctas->into = $6;
 					ctas->objtype = OBJECT_MATVIEW;
 					ctas->is_select_into = false;
 					ctas->if_not_exists = false;
 					/* cram additional flags into the IntoClause */
-					$5->rel->relpersistence = $2;
-					$5->skipData = !($8);
+					$6->rel->relpersistence = $2;
+					$6->skipData = !($9);
+					$6->ivm = $3;
 					$$ = (Node *) ctas;
 				}
-		| CREATE OptNoLog MATERIALIZED VIEW IF_P NOT EXISTS create_mv_target AS SelectStmt opt_with_data
+		| CREATE OptNoLog incremental MATERIALIZED VIEW IF_P NOT EXISTS create_mv_target AS SelectStmt opt_with_data
 				{
 					CreateTableAsStmt *ctas = makeNode(CreateTableAsStmt);
 
-					ctas->query = $10;
-					ctas->into = $8;
+					ctas->query = $11;
+					ctas->into = $9;
 					ctas->objtype = OBJECT_MATVIEW;
 					ctas->is_select_into = false;
 					ctas->if_not_exists = true;
 					/* cram additional flags into the IntoClause */
-					$8->rel->relpersistence = $2;
-					$8->skipData = !($11);
+					$9->rel->relpersistence = $2;
+					$9->skipData = !($12);
+					$9->ivm = $3;
 					$$ = (Node *) ctas;
 				}
 		;
@@ -4738,9 +4741,14 @@ create_mv_target:
 					$$->tableSpaceName = $5;
 					$$->viewQuery = NULL;		/* filled at analysis time */
 					$$->skipData = false;		/* might get changed later */
+					$$->ivm = false;
 				}
 		;
 
+incremental:	INCREMENTAL				{ $$ = true; }
+				| /*EMPTY*/				{ $$ = false; }
+		;
+
 OptNoLog:	UNLOGGED					{ $$ = RELPERSISTENCE_UNLOGGED; }
 			| /*EMPTY*/					{ $$ = RELPERSISTENCE_PERMANENT; }
 		;
@@ -17252,6 +17260,7 @@ unreserved_keyword:
 			| INCLUDE
 			| INCLUDING
 			| INCREMENT
+			| INCREMENTAL
 			| INDENT
 			| INDEX
 			| INDEXES
@@ -17820,6 +17829,7 @@ bare_label_keyword:
 			| INCLUDE
 			| INCLUDING
 			| INCREMENT
+			| INCREMENTAL
 			| INDENT
 			| INDEX
 			| INDEXES
diff --git a/src/include/nodes/primnodes.h b/src/include/nodes/primnodes.h
index 4a154606d2..5d61964f37 100644
--- a/src/include/nodes/primnodes.h
+++ b/src/include/nodes/primnodes.h
@@ -154,6 +154,7 @@ typedef struct IntoClause
 	/* materialized view's SELECT query */
 	Node	   *viewQuery pg_node_attr(query_jumble_ignore);
 	bool		skipData;		/* true for WITH NO DATA */
+	bool		ivm;			/* true for WITH IVM */
 } IntoClause;
 
 
diff --git a/src/include/parser/kwlist.h b/src/include/parser/kwlist.h
index 2331acac09..ab92b54642 100644
--- a/src/include/parser/kwlist.h
+++ b/src/include/parser/kwlist.h
@@ -207,6 +207,7 @@ PG_KEYWORD("in", IN_P, RESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("include", INCLUDE, UNRESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("including", INCLUDING, UNRESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("increment", INCREMENT, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("incremental", INCREMENTAL, UNRESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("indent", INDENT, UNRESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("index", INDEX, UNRESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("indexes", INDEXES, UNRESERVED_KEYWORD, BARE_LABEL)
-- 
2.25.1


--Multipart=_Mon__4_Mar_2024_11_58_46_+0900_UaponF/qQhQrVCFt
Content-Type: text/x-diff;
 name="v30-0002-Add-relisivm-column-to-pg_class-system-catalog.patch"
Content-Disposition: attachment;
 filename="v30-0002-Add-relisivm-column-to-pg_class-system-catalog.patch"
Content-Transfer-Encoding: 7bit



^ permalink  raw  reply  [nested|flat] 125+ messages in thread

* [PATCH v29 01/11] Add a syntax to create Incrementally Maintainable Materialized Views
@ 2019-12-20 01:05 Yugo Nagata <[email protected]>
  0 siblings, 0 replies; 125+ messages in thread

From: Yugo Nagata @ 2019-12-20 01:05 UTC (permalink / raw)

Allow to create Incrementally Maintainable Materialized View (IMMV)
by using INCREMENTAL option in CREATE MATERIALIZED VIEW command
as follow:

     CREATE [INCREMANTAL] MATERIALIZED VIEW xxxxx AS SELECT ....;
---
 src/backend/parser/gram.y     | 32 +++++++++++++++++++++-----------
 src/include/nodes/primnodes.h |  1 +
 src/include/parser/kwlist.h   |  1 +
 3 files changed, 23 insertions(+), 11 deletions(-)

diff --git a/src/backend/parser/gram.y b/src/backend/parser/gram.y
index 7d2032885e..33c647b0c7 100644
--- a/src/backend/parser/gram.y
+++ b/src/backend/parser/gram.y
@@ -465,6 +465,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
 
 %type <range>	OptTempTableName
 %type <into>	into_clause create_as_target create_mv_target
+%type <boolean>	incremental
 
 %type <defelt>	createfunc_opt_item common_func_opt_item dostmt_opt_item
 %type <fun_param> func_arg func_arg_with_default table_func_column aggr_arg
@@ -718,7 +719,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
 	HANDLER HAVING HEADER_P HOLD HOUR_P
 
 	IDENTITY_P IF_P ILIKE IMMEDIATE IMMUTABLE IMPLICIT_P IMPORT_P IN_P INCLUDE
-	INCLUDING INCREMENT INDENT INDEX INDEXES INHERIT INHERITS INITIALLY INLINE_P
+	INCLUDING INCREMENT INCREMENTAL INDENT INDEX INDEXES INHERIT INHERITS INITIALLY INLINE_P
 	INNER_P INOUT INPUT_P INSENSITIVE INSERT INSTEAD INT_P INTEGER
 	INTERSECT INTERVAL INTO INVOKER IS ISNULL ISOLATION
 
@@ -4652,32 +4653,34 @@ opt_with_data:
  *****************************************************************************/
 
 CreateMatViewStmt:
-		CREATE OptNoLog MATERIALIZED VIEW create_mv_target AS SelectStmt opt_with_data
+		CREATE OptNoLog incremental MATERIALIZED VIEW create_mv_target AS SelectStmt opt_with_data
 				{
 					CreateTableAsStmt *ctas = makeNode(CreateTableAsStmt);
 
-					ctas->query = $7;
-					ctas->into = $5;
+					ctas->query = $8;
+					ctas->into = $6;
 					ctas->objtype = OBJECT_MATVIEW;
 					ctas->is_select_into = false;
 					ctas->if_not_exists = false;
 					/* cram additional flags into the IntoClause */
-					$5->rel->relpersistence = $2;
-					$5->skipData = !($8);
+					$6->rel->relpersistence = $2;
+					$6->skipData = !($9);
+					$6->ivm = $3;
 					$$ = (Node *) ctas;
 				}
-		| CREATE OptNoLog MATERIALIZED VIEW IF_P NOT EXISTS create_mv_target AS SelectStmt opt_with_data
+		| CREATE OptNoLog incremental MATERIALIZED VIEW IF_P NOT EXISTS create_mv_target AS SelectStmt opt_with_data
 				{
 					CreateTableAsStmt *ctas = makeNode(CreateTableAsStmt);
 
-					ctas->query = $10;
-					ctas->into = $8;
+					ctas->query = $11;
+					ctas->into = $9;
 					ctas->objtype = OBJECT_MATVIEW;
 					ctas->is_select_into = false;
 					ctas->if_not_exists = true;
 					/* cram additional flags into the IntoClause */
-					$8->rel->relpersistence = $2;
-					$8->skipData = !($11);
+					$9->rel->relpersistence = $2;
+					$9->skipData = !($12);
+					$9->ivm = $3;
 					$$ = (Node *) ctas;
 				}
 		;
@@ -4694,9 +4697,14 @@ create_mv_target:
 					$$->tableSpaceName = $5;
 					$$->viewQuery = NULL;		/* filled at analysis time */
 					$$->skipData = false;		/* might get changed later */
+					$$->ivm = false;
 				}
 		;
 
+incremental:	INCREMENTAL				{ $$ = true; }
+				| /*EMPTY*/				{ $$ = false; }
+		;
+
 OptNoLog:	UNLOGGED					{ $$ = RELPERSISTENCE_UNLOGGED; }
 			| /*EMPTY*/					{ $$ = RELPERSISTENCE_PERMANENT; }
 		;
@@ -17141,6 +17149,7 @@ unreserved_keyword:
 			| INCLUDE
 			| INCLUDING
 			| INCREMENT
+			| INCREMENTAL
 			| INDENT
 			| INDEX
 			| INDEXES
@@ -17709,6 +17718,7 @@ bare_label_keyword:
 			| INCLUDE
 			| INCLUDING
 			| INCREMENT
+			| INCREMENTAL
 			| INDENT
 			| INDEX
 			| INDEXES
diff --git a/src/include/nodes/primnodes.h b/src/include/nodes/primnodes.h
index 60d72a876b..cecb968b36 100644
--- a/src/include/nodes/primnodes.h
+++ b/src/include/nodes/primnodes.h
@@ -146,6 +146,7 @@ typedef struct IntoClause
 	/* materialized view's SELECT query */
 	Node	   *viewQuery pg_node_attr(query_jumble_ignore);
 	bool		skipData;		/* true for WITH NO DATA */
+	bool		ivm;			/* true for WITH IVM */
 } IntoClause;
 
 
diff --git a/src/include/parser/kwlist.h b/src/include/parser/kwlist.h
index 5984dcfa4b..d60eb98d65 100644
--- a/src/include/parser/kwlist.h
+++ b/src/include/parser/kwlist.h
@@ -207,6 +207,7 @@ PG_KEYWORD("in", IN_P, RESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("include", INCLUDE, UNRESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("including", INCLUDING, UNRESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("increment", INCREMENT, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("incremental", INCREMENTAL, UNRESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("indent", INDENT, UNRESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("index", INDEX, UNRESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("indexes", INDEXES, UNRESERVED_KEYWORD, BARE_LABEL)
-- 
2.25.1


--Multipart=_Mon__28_Aug_2023_16_05_30_+0900_b1OvQD_3A3ZMTGvj
Content-Type: text/x-diff;
 name="v29-0002-Add-relisivm-column-to-pg_class-system-catalog.patch"
Content-Disposition: attachment;
 filename="v29-0002-Add-relisivm-column-to-pg_class-system-catalog.patch"
Content-Transfer-Encoding: 7bit



^ permalink  raw  reply  [nested|flat] 125+ messages in thread

* [PATCH v27 1/9] Add a syntax to create Incrementally Maintainable Materialized Views
@ 2019-12-20 01:05 Yugo Nagata <[email protected]>
  0 siblings, 0 replies; 125+ messages in thread

From: Yugo Nagata @ 2019-12-20 01:05 UTC (permalink / raw)

Allow to create Incrementally Maintainable Materialized View (IMMV)
by using INCREMENTAL option in CREATE MATERIALIZED VIEW command
as follow:

     CREATE [INCREMANTAL] MATERIALIZED VIEW xxxxx AS SELECT ....;
---
 src/backend/nodes/copyfuncs.c  |  1 +
 src/backend/nodes/equalfuncs.c |  1 +
 src/backend/nodes/outfuncs.c   |  1 +
 src/backend/nodes/readfuncs.c  |  1 +
 src/backend/parser/gram.y      | 32 +++++++++++++++++++++-----------
 src/include/nodes/primnodes.h  |  1 +
 src/include/parser/kwlist.h    |  1 +
 7 files changed, 27 insertions(+), 11 deletions(-)

diff --git a/src/backend/nodes/copyfuncs.c b/src/backend/nodes/copyfuncs.c
index 836f427ea8..0490bce664 100644
--- a/src/backend/nodes/copyfuncs.c
+++ b/src/backend/nodes/copyfuncs.c
@@ -1434,6 +1434,7 @@ _copyIntoClause(const IntoClause *from)
 	COPY_STRING_FIELD(tableSpaceName);
 	COPY_NODE_FIELD(viewQuery);
 	COPY_SCALAR_FIELD(skipData);
+	COPY_SCALAR_FIELD(ivm);
 
 	return newnode;
 }
diff --git a/src/backend/nodes/equalfuncs.c b/src/backend/nodes/equalfuncs.c
index e013c1bbfe..55f41263ee 100644
--- a/src/backend/nodes/equalfuncs.c
+++ b/src/backend/nodes/equalfuncs.c
@@ -211,6 +211,7 @@ _equalIntoClause(const IntoClause *a, const IntoClause *b)
 	COMPARE_STRING_FIELD(tableSpaceName);
 	COMPARE_NODE_FIELD(viewQuery);
 	COMPARE_SCALAR_FIELD(skipData);
+	COMPARE_SCALAR_FIELD(ivm);
 
 	return true;
 }
diff --git a/src/backend/nodes/outfuncs.c b/src/backend/nodes/outfuncs.c
index 6a02f81ad5..cfd3ce68b4 100644
--- a/src/backend/nodes/outfuncs.c
+++ b/src/backend/nodes/outfuncs.c
@@ -1127,6 +1127,7 @@ _outIntoClause(StringInfo str, const IntoClause *node)
 	WRITE_STRING_FIELD(tableSpaceName);
 	WRITE_NODE_FIELD(viewQuery);
 	WRITE_BOOL_FIELD(skipData);
+	WRITE_BOOL_FIELD(ivm);
 }
 
 static void
diff --git a/src/backend/nodes/readfuncs.c b/src/backend/nodes/readfuncs.c
index ddf76ac778..5165fb3b93 100644
--- a/src/backend/nodes/readfuncs.c
+++ b/src/backend/nodes/readfuncs.c
@@ -605,6 +605,7 @@ _readIntoClause(void)
 	READ_STRING_FIELD(tableSpaceName);
 	READ_NODE_FIELD(viewQuery);
 	READ_BOOL_FIELD(skipData);
+	READ_BOOL_FIELD(ivm);
 
 	READ_DONE();
 }
diff --git a/src/backend/parser/gram.y b/src/backend/parser/gram.y
index c9941d9cb4..aa134eafc9 100644
--- a/src/backend/parser/gram.y
+++ b/src/backend/parser/gram.y
@@ -468,6 +468,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
 
 %type <range>	OptTempTableName
 %type <into>	into_clause create_as_target create_mv_target
+%type <boolean>	incremental
 
 %type <defelt>	createfunc_opt_item common_func_opt_item dostmt_opt_item
 %type <fun_param> func_arg func_arg_with_default table_func_column aggr_arg
@@ -801,7 +802,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
 	HANDLER HAVING HEADER_P HOLD HOUR_P
 
 	IDENTITY_P IF_P ILIKE IMMEDIATE IMMUTABLE IMPLICIT_P IMPORT_P IN_P INCLUDE
-	INCLUDING INCREMENT INDEX INDEXES INHERIT INHERITS INITIALLY INLINE_P
+	INCLUDING INCREMENT INCREMENTAL INDEX INDEXES INHERIT INHERITS INITIALLY INLINE_P
 	INNER_P INOUT INPUT_P INSENSITIVE INSERT INSTEAD INT_P INTEGER
 	INTERSECT INTERVAL INTO INVOKER IS ISNULL ISOLATION
 
@@ -4492,30 +4493,32 @@ opt_with_data:
  *****************************************************************************/
 
 CreateMatViewStmt:
-		CREATE OptNoLog MATERIALIZED VIEW create_mv_target AS SelectStmt opt_with_data
+		CREATE OptNoLog incremental MATERIALIZED VIEW create_mv_target AS SelectStmt opt_with_data
 				{
 					CreateTableAsStmt *ctas = makeNode(CreateTableAsStmt);
-					ctas->query = $7;
-					ctas->into = $5;
+					ctas->query = $8;
+					ctas->into = $6;
 					ctas->objtype = OBJECT_MATVIEW;
 					ctas->is_select_into = false;
 					ctas->if_not_exists = false;
 					/* cram additional flags into the IntoClause */
-					$5->rel->relpersistence = $2;
-					$5->skipData = !($8);
+					$6->rel->relpersistence = $2;
+					$6->skipData = !($9);
+					$6->ivm = $3;
 					$$ = (Node *) ctas;
 				}
-		| CREATE OptNoLog MATERIALIZED VIEW IF_P NOT EXISTS create_mv_target AS SelectStmt opt_with_data
+		| CREATE OptNoLog incremental MATERIALIZED VIEW IF_P NOT EXISTS create_mv_target AS SelectStmt opt_with_data
 				{
 					CreateTableAsStmt *ctas = makeNode(CreateTableAsStmt);
-					ctas->query = $10;
-					ctas->into = $8;
+					ctas->query = $11;
+					ctas->into = $9;
 					ctas->objtype = OBJECT_MATVIEW;
 					ctas->is_select_into = false;
 					ctas->if_not_exists = true;
 					/* cram additional flags into the IntoClause */
-					$8->rel->relpersistence = $2;
-					$8->skipData = !($11);
+					$9->rel->relpersistence = $2;
+					$9->skipData = !($12);
+					$9->ivm = $3;
 					$$ = (Node *) ctas;
 				}
 		;
@@ -4532,9 +4535,14 @@ create_mv_target:
 					$$->tableSpaceName = $5;
 					$$->viewQuery = NULL;		/* filled at analysis time */
 					$$->skipData = false;		/* might get changed later */
+					$$->ivm = false;
 				}
 		;
 
+incremental:	INCREMENTAL				{ $$ = true; }
+				| /*EMPTY*/				{ $$ = false; }
+		;
+
 OptNoLog:	UNLOGGED					{ $$ = RELPERSISTENCE_UNLOGGED; }
 			| /*EMPTY*/					{ $$ = RELPERSISTENCE_PERMANENT; }
 		;
@@ -17007,6 +17015,7 @@ unreserved_keyword:
 			| INCLUDE
 			| INCLUDING
 			| INCREMENT
+			| INCREMENTAL
 			| INDEX
 			| INDEXES
 			| INHERIT
@@ -17588,6 +17597,7 @@ bare_label_keyword:
 			| INCLUDE
 			| INCLUDING
 			| INCREMENT
+			| INCREMENTAL
 			| INDEX
 			| INDEXES
 			| INHERIT
diff --git a/src/include/nodes/primnodes.h b/src/include/nodes/primnodes.h
index 66d32fc006..acac38b91c 100644
--- a/src/include/nodes/primnodes.h
+++ b/src/include/nodes/primnodes.h
@@ -126,6 +126,7 @@ typedef struct IntoClause
 	char	   *tableSpaceName; /* table space to use, or NULL */
 	Node	   *viewQuery;		/* materialized view's SELECT query */
 	bool		skipData;		/* true for WITH NO DATA */
+	bool		ivm;			/* true for WITH IVM */
 } IntoClause;
 
 
diff --git a/src/include/parser/kwlist.h b/src/include/parser/kwlist.h
index 8a2ab405a2..7cd9b54a01 100644
--- a/src/include/parser/kwlist.h
+++ b/src/include/parser/kwlist.h
@@ -210,6 +210,7 @@ PG_KEYWORD("in", IN_P, RESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("include", INCLUDE, UNRESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("including", INCLUDING, UNRESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("increment", INCREMENT, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("incremental", INCREMENTAL, UNRESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("index", INDEX, UNRESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("indexes", INDEXES, UNRESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("inherit", INHERIT, UNRESERVED_KEYWORD, BARE_LABEL)
-- 
2.17.1


--Multipart=_Fri__22_Apr_2022_11_29_39_+0900_ZOAC7UMt5e8j1Nvx
Content-Type: text/x-diff;
 name="v27-0002-Add-relisivm-column-to-pg_class-system-catalog.patch"
Content-Disposition: attachment;
 filename="v27-0002-Add-relisivm-column-to-pg_class-system-catalog.patch"
Content-Transfer-Encoding: 7bit



^ permalink  raw  reply  [nested|flat] 125+ messages in thread

* [PATCH v39 1/8] Add a syntax to create Incrementally Maintainable Materialized Views
@ 2019-12-20 01:05 Yugo Nagata <[email protected]>
  0 siblings, 0 replies; 125+ messages in thread

From: Yugo Nagata @ 2019-12-20 01:05 UTC (permalink / raw)

Allow to create Incrementally Maintainable Materialized View (IMMV)
by using INCREMENTAL option in CREATE MATERIALIZED VIEW command
as follow:

     CREATE [INCREMANTAL] MATERIALIZED VIEW xxxxx AS SELECT ....;
---
 src/backend/parser/gram.y     | 32 +++++++++++++++++++++-----------
 src/include/nodes/primnodes.h |  1 +
 src/include/parser/kwlist.h   |  1 +
 3 files changed, 23 insertions(+), 11 deletions(-)

diff --git a/src/backend/parser/gram.y b/src/backend/parser/gram.y
index ff4e1388c55..9a353550c37 100644
--- a/src/backend/parser/gram.y
+++ b/src/backend/parser/gram.y
@@ -473,6 +473,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
 
 %type <range>	OptTempTableName
 %type <into>	into_clause create_as_target create_mv_target
+%type <boolean>	incremental
 
 %type <defelt>	createfunc_opt_item common_func_opt_item dostmt_opt_item
 %type <fun_param> func_arg func_arg_with_default table_func_column aggr_arg
@@ -776,7 +777,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
 	HANDLER HAVING HEADER_P HOLD HOUR_P
 
 	IDENTITY_P IF_P IGNORE_P ILIKE IMMEDIATE IMMUTABLE IMPLICIT_P IMPORT_P IN_P INCLUDE
-	INCLUDING INCREMENT INDENT INDEX INDEXES INHERIT INHERITS INITIALLY INLINE_P
+	INCLUDING INCREMENT INCREMENTAL INDENT INDEX INDEXES INHERIT INHERITS INITIALLY INLINE_P
 	INNER_P INOUT INPUT_P INSENSITIVE INSERT INSTEAD INT_P INTEGER
 	INTERSECT INTERVAL INTO INVOKER IS ISNULL ISOLATION
 
@@ -5039,32 +5040,34 @@ opt_with_data:
  *****************************************************************************/
 
 CreateMatViewStmt:
-		CREATE OptNoLog MATERIALIZED VIEW create_mv_target AS SelectStmt opt_with_data
+		CREATE OptNoLog incremental MATERIALIZED VIEW create_mv_target AS SelectStmt opt_with_data
 				{
 					CreateTableAsStmt *ctas = makeNode(CreateTableAsStmt);
 
-					ctas->query = $7;
-					ctas->into = $5;
+					ctas->query = $8;
+					ctas->into = $6;
 					ctas->objtype = OBJECT_MATVIEW;
 					ctas->is_select_into = false;
 					ctas->if_not_exists = false;
 					/* cram additional flags into the IntoClause */
-					$5->rel->relpersistence = $2;
-					$5->skipData = !($8);
+					$6->rel->relpersistence = $2;
+					$6->skipData = !($9);
+					$6->ivm = $3;
 					$$ = (Node *) ctas;
 				}
-		| CREATE OptNoLog MATERIALIZED VIEW IF_P NOT EXISTS create_mv_target AS SelectStmt opt_with_data
+		| CREATE OptNoLog incremental MATERIALIZED VIEW IF_P NOT EXISTS create_mv_target AS SelectStmt opt_with_data
 				{
 					CreateTableAsStmt *ctas = makeNode(CreateTableAsStmt);
 
-					ctas->query = $10;
-					ctas->into = $8;
+					ctas->query = $11;
+					ctas->into = $9;
 					ctas->objtype = OBJECT_MATVIEW;
 					ctas->is_select_into = false;
 					ctas->if_not_exists = true;
 					/* cram additional flags into the IntoClause */
-					$8->rel->relpersistence = $2;
-					$8->skipData = !($11);
+					$9->rel->relpersistence = $2;
+					$9->skipData = !($12);
+					$9->ivm = $3;
 					$$ = (Node *) ctas;
 				}
 		;
@@ -5081,9 +5084,14 @@ create_mv_target:
 					$$->tableSpaceName = $5;
 					$$->viewQuery = NULL;		/* filled at analysis time */
 					$$->skipData = false;		/* might get changed later */
+					$$->ivm = false;
 				}
 		;
 
+incremental:	INCREMENTAL				{ $$ = true; }
+				| /*EMPTY*/				{ $$ = false; }
+		;
+
 OptNoLog:	UNLOGGED					{ $$ = RELPERSISTENCE_UNLOGGED; }
 			| /*EMPTY*/					{ $$ = RELPERSISTENCE_PERMANENT; }
 		;
@@ -18948,6 +18956,7 @@ unreserved_keyword:
 			| INCLUDE
 			| INCLUDING
 			| INCREMENT
+			| INCREMENTAL
 			| INDENT
 			| INDEX
 			| INDEXES
@@ -19554,6 +19563,7 @@ bare_label_keyword:
 			| INCLUDE
 			| INCLUDING
 			| INCREMENT
+			| INCREMENTAL
 			| INDENT
 			| INDEX
 			| INDEXES
diff --git a/src/include/nodes/primnodes.h b/src/include/nodes/primnodes.h
index bb05aeebee4..da35b15cc27 100644
--- a/src/include/nodes/primnodes.h
+++ b/src/include/nodes/primnodes.h
@@ -170,6 +170,7 @@ typedef struct IntoClause
 	/* materialized view's SELECT query */
 	struct Query *viewQuery pg_node_attr(query_jumble_ignore);
 	bool		skipData;		/* true for WITH NO DATA */
+	bool		ivm;			/* true for WITH IVM */
 } IntoClause;
 
 
diff --git a/src/include/parser/kwlist.h b/src/include/parser/kwlist.h
index 51ead54f015..7849ee7f960 100644
--- a/src/include/parser/kwlist.h
+++ b/src/include/parser/kwlist.h
@@ -216,6 +216,7 @@ PG_KEYWORD("in", IN_P, RESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("include", INCLUDE, UNRESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("including", INCLUDING, UNRESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("increment", INCREMENT, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("incremental", INCREMENTAL, UNRESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("indent", INDENT, UNRESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("index", INDEX, UNRESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("indexes", INDEXES, UNRESERVED_KEYWORD, BARE_LABEL)
-- 
2.43.0


--Multipart=_Fri__3_Jul_2026_19_11_16_+0900_CskxSIu_iGcDMEyM--






^ permalink  raw  reply  [nested|flat] 125+ messages in thread

* [PATCH v27 1/9] Add a syntax to create Incrementally Maintainable Materialized Views
@ 2019-12-20 01:05 Yugo Nagata <[email protected]>
  0 siblings, 0 replies; 125+ messages in thread

From: Yugo Nagata @ 2019-12-20 01:05 UTC (permalink / raw)

Allow to create Incrementally Maintainable Materialized View (IMMV)
by using INCREMENTAL option in CREATE MATERIALIZED VIEW command
as follow:

     CREATE [INCREMANTAL] MATERIALIZED VIEW xxxxx AS SELECT ....;
---
 src/backend/nodes/copyfuncs.c  |  1 +
 src/backend/nodes/equalfuncs.c |  1 +
 src/backend/nodes/outfuncs.c   |  1 +
 src/backend/nodes/readfuncs.c  |  1 +
 src/backend/parser/gram.y      | 32 +++++++++++++++++++++-----------
 src/include/nodes/primnodes.h  |  1 +
 src/include/parser/kwlist.h    |  1 +
 7 files changed, 27 insertions(+), 11 deletions(-)

diff --git a/src/backend/nodes/copyfuncs.c b/src/backend/nodes/copyfuncs.c
index 836f427ea8..0490bce664 100644
--- a/src/backend/nodes/copyfuncs.c
+++ b/src/backend/nodes/copyfuncs.c
@@ -1434,6 +1434,7 @@ _copyIntoClause(const IntoClause *from)
 	COPY_STRING_FIELD(tableSpaceName);
 	COPY_NODE_FIELD(viewQuery);
 	COPY_SCALAR_FIELD(skipData);
+	COPY_SCALAR_FIELD(ivm);
 
 	return newnode;
 }
diff --git a/src/backend/nodes/equalfuncs.c b/src/backend/nodes/equalfuncs.c
index e013c1bbfe..55f41263ee 100644
--- a/src/backend/nodes/equalfuncs.c
+++ b/src/backend/nodes/equalfuncs.c
@@ -211,6 +211,7 @@ _equalIntoClause(const IntoClause *a, const IntoClause *b)
 	COMPARE_STRING_FIELD(tableSpaceName);
 	COMPARE_NODE_FIELD(viewQuery);
 	COMPARE_SCALAR_FIELD(skipData);
+	COMPARE_SCALAR_FIELD(ivm);
 
 	return true;
 }
diff --git a/src/backend/nodes/outfuncs.c b/src/backend/nodes/outfuncs.c
index 6a02f81ad5..cfd3ce68b4 100644
--- a/src/backend/nodes/outfuncs.c
+++ b/src/backend/nodes/outfuncs.c
@@ -1127,6 +1127,7 @@ _outIntoClause(StringInfo str, const IntoClause *node)
 	WRITE_STRING_FIELD(tableSpaceName);
 	WRITE_NODE_FIELD(viewQuery);
 	WRITE_BOOL_FIELD(skipData);
+	WRITE_BOOL_FIELD(ivm);
 }
 
 static void
diff --git a/src/backend/nodes/readfuncs.c b/src/backend/nodes/readfuncs.c
index ddf76ac778..5165fb3b93 100644
--- a/src/backend/nodes/readfuncs.c
+++ b/src/backend/nodes/readfuncs.c
@@ -605,6 +605,7 @@ _readIntoClause(void)
 	READ_STRING_FIELD(tableSpaceName);
 	READ_NODE_FIELD(viewQuery);
 	READ_BOOL_FIELD(skipData);
+	READ_BOOL_FIELD(ivm);
 
 	READ_DONE();
 }
diff --git a/src/backend/parser/gram.y b/src/backend/parser/gram.y
index c9941d9cb4..aa134eafc9 100644
--- a/src/backend/parser/gram.y
+++ b/src/backend/parser/gram.y
@@ -468,6 +468,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
 
 %type <range>	OptTempTableName
 %type <into>	into_clause create_as_target create_mv_target
+%type <boolean>	incremental
 
 %type <defelt>	createfunc_opt_item common_func_opt_item dostmt_opt_item
 %type <fun_param> func_arg func_arg_with_default table_func_column aggr_arg
@@ -801,7 +802,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
 	HANDLER HAVING HEADER_P HOLD HOUR_P
 
 	IDENTITY_P IF_P ILIKE IMMEDIATE IMMUTABLE IMPLICIT_P IMPORT_P IN_P INCLUDE
-	INCLUDING INCREMENT INDEX INDEXES INHERIT INHERITS INITIALLY INLINE_P
+	INCLUDING INCREMENT INCREMENTAL INDEX INDEXES INHERIT INHERITS INITIALLY INLINE_P
 	INNER_P INOUT INPUT_P INSENSITIVE INSERT INSTEAD INT_P INTEGER
 	INTERSECT INTERVAL INTO INVOKER IS ISNULL ISOLATION
 
@@ -4492,30 +4493,32 @@ opt_with_data:
  *****************************************************************************/
 
 CreateMatViewStmt:
-		CREATE OptNoLog MATERIALIZED VIEW create_mv_target AS SelectStmt opt_with_data
+		CREATE OptNoLog incremental MATERIALIZED VIEW create_mv_target AS SelectStmt opt_with_data
 				{
 					CreateTableAsStmt *ctas = makeNode(CreateTableAsStmt);
-					ctas->query = $7;
-					ctas->into = $5;
+					ctas->query = $8;
+					ctas->into = $6;
 					ctas->objtype = OBJECT_MATVIEW;
 					ctas->is_select_into = false;
 					ctas->if_not_exists = false;
 					/* cram additional flags into the IntoClause */
-					$5->rel->relpersistence = $2;
-					$5->skipData = !($8);
+					$6->rel->relpersistence = $2;
+					$6->skipData = !($9);
+					$6->ivm = $3;
 					$$ = (Node *) ctas;
 				}
-		| CREATE OptNoLog MATERIALIZED VIEW IF_P NOT EXISTS create_mv_target AS SelectStmt opt_with_data
+		| CREATE OptNoLog incremental MATERIALIZED VIEW IF_P NOT EXISTS create_mv_target AS SelectStmt opt_with_data
 				{
 					CreateTableAsStmt *ctas = makeNode(CreateTableAsStmt);
-					ctas->query = $10;
-					ctas->into = $8;
+					ctas->query = $11;
+					ctas->into = $9;
 					ctas->objtype = OBJECT_MATVIEW;
 					ctas->is_select_into = false;
 					ctas->if_not_exists = true;
 					/* cram additional flags into the IntoClause */
-					$8->rel->relpersistence = $2;
-					$8->skipData = !($11);
+					$9->rel->relpersistence = $2;
+					$9->skipData = !($12);
+					$9->ivm = $3;
 					$$ = (Node *) ctas;
 				}
 		;
@@ -4532,9 +4535,14 @@ create_mv_target:
 					$$->tableSpaceName = $5;
 					$$->viewQuery = NULL;		/* filled at analysis time */
 					$$->skipData = false;		/* might get changed later */
+					$$->ivm = false;
 				}
 		;
 
+incremental:	INCREMENTAL				{ $$ = true; }
+				| /*EMPTY*/				{ $$ = false; }
+		;
+
 OptNoLog:	UNLOGGED					{ $$ = RELPERSISTENCE_UNLOGGED; }
 			| /*EMPTY*/					{ $$ = RELPERSISTENCE_PERMANENT; }
 		;
@@ -17007,6 +17015,7 @@ unreserved_keyword:
 			| INCLUDE
 			| INCLUDING
 			| INCREMENT
+			| INCREMENTAL
 			| INDEX
 			| INDEXES
 			| INHERIT
@@ -17588,6 +17597,7 @@ bare_label_keyword:
 			| INCLUDE
 			| INCLUDING
 			| INCREMENT
+			| INCREMENTAL
 			| INDEX
 			| INDEXES
 			| INHERIT
diff --git a/src/include/nodes/primnodes.h b/src/include/nodes/primnodes.h
index 66d32fc006..acac38b91c 100644
--- a/src/include/nodes/primnodes.h
+++ b/src/include/nodes/primnodes.h
@@ -126,6 +126,7 @@ typedef struct IntoClause
 	char	   *tableSpaceName; /* table space to use, or NULL */
 	Node	   *viewQuery;		/* materialized view's SELECT query */
 	bool		skipData;		/* true for WITH NO DATA */
+	bool		ivm;			/* true for WITH IVM */
 } IntoClause;
 
 
diff --git a/src/include/parser/kwlist.h b/src/include/parser/kwlist.h
index 8a2ab405a2..7cd9b54a01 100644
--- a/src/include/parser/kwlist.h
+++ b/src/include/parser/kwlist.h
@@ -210,6 +210,7 @@ PG_KEYWORD("in", IN_P, RESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("include", INCLUDE, UNRESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("including", INCLUDING, UNRESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("increment", INCREMENT, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("incremental", INCREMENTAL, UNRESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("index", INDEX, UNRESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("indexes", INDEXES, UNRESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("inherit", INHERIT, UNRESERVED_KEYWORD, BARE_LABEL)
-- 
2.17.1


--Multipart=_Fri__22_Apr_2022_14_58_01_+0900_MN3L/o2YUVF2g4zw
Content-Type: text/x-diff;
 name="v27-0002-Add-relisivm-column-to-pg_class-system-catalog.patch"
Content-Disposition: attachment;
 filename="v27-0002-Add-relisivm-column-to-pg_class-system-catalog.patch"
Content-Transfer-Encoding: 7bit



^ permalink  raw  reply  [nested|flat] 125+ messages in thread

* [PATCH v31 01/11] Add a syntax to create Incrementally Maintainable Materialized Views
@ 2019-12-20 01:05 Yugo Nagata <[email protected]>
  0 siblings, 0 replies; 125+ messages in thread

From: Yugo Nagata @ 2019-12-20 01:05 UTC (permalink / raw)

Allow to create Incrementally Maintainable Materialized View (IMMV)
by using INCREMENTAL option in CREATE MATERIALIZED VIEW command
as follow:

     CREATE [INCREMANTAL] MATERIALIZED VIEW xxxxx AS SELECT ....;
---
 src/backend/parser/gram.y     | 32 +++++++++++++++++++++-----------
 src/include/nodes/primnodes.h |  1 +
 src/include/parser/kwlist.h   |  1 +
 3 files changed, 23 insertions(+), 11 deletions(-)

diff --git a/src/backend/parser/gram.y b/src/backend/parser/gram.y
index c1b0cff1c9..3a4746a22b 100644
--- a/src/backend/parser/gram.y
+++ b/src/backend/parser/gram.y
@@ -467,6 +467,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
 
 %type <range>	OptTempTableName
 %type <into>	into_clause create_as_target create_mv_target
+%type <boolean>	incremental
 
 %type <defelt>	createfunc_opt_item common_func_opt_item dostmt_opt_item
 %type <fun_param> func_arg func_arg_with_default table_func_column aggr_arg
@@ -730,7 +731,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
 	HANDLER HAVING HEADER_P HOLD HOUR_P
 
 	IDENTITY_P IF_P ILIKE IMMEDIATE IMMUTABLE IMPLICIT_P IMPORT_P IN_P INCLUDE
-	INCLUDING INCREMENT INDENT INDEX INDEXES INHERIT INHERITS INITIALLY INLINE_P
+	INCLUDING INCREMENT INCREMENTAL INDENT INDEX INDEXES INHERIT INHERITS INITIALLY INLINE_P
 	INNER_P INOUT INPUT_P INSENSITIVE INSERT INSTEAD INT_P INTEGER
 	INTERSECT INTERVAL INTO INVOKER IS ISNULL ISOLATION
 
@@ -4732,32 +4733,34 @@ opt_with_data:
  *****************************************************************************/
 
 CreateMatViewStmt:
-		CREATE OptNoLog MATERIALIZED VIEW create_mv_target AS SelectStmt opt_with_data
+		CREATE OptNoLog incremental MATERIALIZED VIEW create_mv_target AS SelectStmt opt_with_data
 				{
 					CreateTableAsStmt *ctas = makeNode(CreateTableAsStmt);
 
-					ctas->query = $7;
-					ctas->into = $5;
+					ctas->query = $8;
+					ctas->into = $6;
 					ctas->objtype = OBJECT_MATVIEW;
 					ctas->is_select_into = false;
 					ctas->if_not_exists = false;
 					/* cram additional flags into the IntoClause */
-					$5->rel->relpersistence = $2;
-					$5->skipData = !($8);
+					$6->rel->relpersistence = $2;
+					$6->skipData = !($9);
+					$6->ivm = $3;
 					$$ = (Node *) ctas;
 				}
-		| CREATE OptNoLog MATERIALIZED VIEW IF_P NOT EXISTS create_mv_target AS SelectStmt opt_with_data
+		| CREATE OptNoLog incremental MATERIALIZED VIEW IF_P NOT EXISTS create_mv_target AS SelectStmt opt_with_data
 				{
 					CreateTableAsStmt *ctas = makeNode(CreateTableAsStmt);
 
-					ctas->query = $10;
-					ctas->into = $8;
+					ctas->query = $11;
+					ctas->into = $9;
 					ctas->objtype = OBJECT_MATVIEW;
 					ctas->is_select_into = false;
 					ctas->if_not_exists = true;
 					/* cram additional flags into the IntoClause */
-					$8->rel->relpersistence = $2;
-					$8->skipData = !($11);
+					$9->rel->relpersistence = $2;
+					$9->skipData = !($12);
+					$9->ivm = $3;
 					$$ = (Node *) ctas;
 				}
 		;
@@ -4774,9 +4777,14 @@ create_mv_target:
 					$$->tableSpaceName = $5;
 					$$->viewQuery = NULL;		/* filled at analysis time */
 					$$->skipData = false;		/* might get changed later */
+					$$->ivm = false;
 				}
 		;
 
+incremental:	INCREMENTAL				{ $$ = true; }
+				| /*EMPTY*/				{ $$ = false; }
+		;
+
 OptNoLog:	UNLOGGED					{ $$ = RELPERSISTENCE_UNLOGGED; }
 			| /*EMPTY*/					{ $$ = RELPERSISTENCE_PERMANENT; }
 		;
@@ -17436,6 +17444,7 @@ unreserved_keyword:
 			| INCLUDE
 			| INCLUDING
 			| INCREMENT
+			| INCREMENTAL
 			| INDENT
 			| INDEX
 			| INDEXES
@@ -18017,6 +18026,7 @@ bare_label_keyword:
 			| INCLUDE
 			| INCLUDING
 			| INCREMENT
+			| INCREMENTAL
 			| INDENT
 			| INDEX
 			| INDEXES
diff --git a/src/include/nodes/primnodes.h b/src/include/nodes/primnodes.h
index 376f67e6a5..f7bcf78cf2 100644
--- a/src/include/nodes/primnodes.h
+++ b/src/include/nodes/primnodes.h
@@ -154,6 +154,7 @@ typedef struct IntoClause
 	/* materialized view's SELECT query */
 	Node	   *viewQuery pg_node_attr(query_jumble_ignore);
 	bool		skipData;		/* true for WITH NO DATA */
+	bool		ivm;			/* true for WITH IVM */
 } IntoClause;
 
 
diff --git a/src/include/parser/kwlist.h b/src/include/parser/kwlist.h
index 57514d064b..f146d7c41a 100644
--- a/src/include/parser/kwlist.h
+++ b/src/include/parser/kwlist.h
@@ -210,6 +210,7 @@ PG_KEYWORD("in", IN_P, RESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("include", INCLUDE, UNRESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("including", INCLUDING, UNRESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("increment", INCREMENT, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("incremental", INCREMENTAL, UNRESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("indent", INDENT, UNRESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("index", INDEX, UNRESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("indexes", INDEXES, UNRESERVED_KEYWORD, BARE_LABEL)
-- 
2.25.1


--Multipart=_Fri__29_Mar_2024_23_47_00_+0900_KGpmmDOIs1266Ib1
Content-Type: text/x-diff;
 name="v31-0002-Add-relisivm-column-to-pg_class-system-catalog.patch"
Content-Disposition: attachment;
 filename="v31-0002-Add-relisivm-column-to-pg_class-system-catalog.patch"
Content-Transfer-Encoding: 7bit



^ permalink  raw  reply  [nested|flat] 125+ messages in thread

* [PATCH v30 01/11] Add a syntax to create Incrementally Maintainable Materialized Views
@ 2019-12-20 01:05 Yugo Nagata <[email protected]>
  0 siblings, 0 replies; 125+ messages in thread

From: Yugo Nagata @ 2019-12-20 01:05 UTC (permalink / raw)

Allow to create Incrementally Maintainable Materialized View (IMMV)
by using INCREMENTAL option in CREATE MATERIALIZED VIEW command
as follow:

     CREATE [INCREMANTAL] MATERIALIZED VIEW xxxxx AS SELECT ....;
---
 src/backend/parser/gram.y     | 32 +++++++++++++++++++++-----------
 src/include/nodes/primnodes.h |  1 +
 src/include/parser/kwlist.h   |  1 +
 3 files changed, 23 insertions(+), 11 deletions(-)

diff --git a/src/backend/parser/gram.y b/src/backend/parser/gram.y
index 130f7fc7c3..7ba972ae5d 100644
--- a/src/backend/parser/gram.y
+++ b/src/backend/parser/gram.y
@@ -466,6 +466,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
 
 %type <range>	OptTempTableName
 %type <into>	into_clause create_as_target create_mv_target
+%type <boolean>	incremental
 
 %type <defelt>	createfunc_opt_item common_func_opt_item dostmt_opt_item
 %type <fun_param> func_arg func_arg_with_default table_func_column aggr_arg
@@ -719,7 +720,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
 	HANDLER HAVING HEADER_P HOLD HOUR_P
 
 	IDENTITY_P IF_P ILIKE IMMEDIATE IMMUTABLE IMPLICIT_P IMPORT_P IN_P INCLUDE
-	INCLUDING INCREMENT INDENT INDEX INDEXES INHERIT INHERITS INITIALLY INLINE_P
+	INCLUDING INCREMENT INCREMENTAL INDENT INDEX INDEXES INHERIT INHERITS INITIALLY INLINE_P
 	INNER_P INOUT INPUT_P INSENSITIVE INSERT INSTEAD INT_P INTEGER
 	INTERSECT INTERVAL INTO INVOKER IS ISNULL ISOLATION
 
@@ -4696,32 +4697,34 @@ opt_with_data:
  *****************************************************************************/
 
 CreateMatViewStmt:
-		CREATE OptNoLog MATERIALIZED VIEW create_mv_target AS SelectStmt opt_with_data
+		CREATE OptNoLog incremental MATERIALIZED VIEW create_mv_target AS SelectStmt opt_with_data
 				{
 					CreateTableAsStmt *ctas = makeNode(CreateTableAsStmt);
 
-					ctas->query = $7;
-					ctas->into = $5;
+					ctas->query = $8;
+					ctas->into = $6;
 					ctas->objtype = OBJECT_MATVIEW;
 					ctas->is_select_into = false;
 					ctas->if_not_exists = false;
 					/* cram additional flags into the IntoClause */
-					$5->rel->relpersistence = $2;
-					$5->skipData = !($8);
+					$6->rel->relpersistence = $2;
+					$6->skipData = !($9);
+					$6->ivm = $3;
 					$$ = (Node *) ctas;
 				}
-		| CREATE OptNoLog MATERIALIZED VIEW IF_P NOT EXISTS create_mv_target AS SelectStmt opt_with_data
+		| CREATE OptNoLog incremental MATERIALIZED VIEW IF_P NOT EXISTS create_mv_target AS SelectStmt opt_with_data
 				{
 					CreateTableAsStmt *ctas = makeNode(CreateTableAsStmt);
 
-					ctas->query = $10;
-					ctas->into = $8;
+					ctas->query = $11;
+					ctas->into = $9;
 					ctas->objtype = OBJECT_MATVIEW;
 					ctas->is_select_into = false;
 					ctas->if_not_exists = true;
 					/* cram additional flags into the IntoClause */
-					$8->rel->relpersistence = $2;
-					$8->skipData = !($11);
+					$9->rel->relpersistence = $2;
+					$9->skipData = !($12);
+					$9->ivm = $3;
 					$$ = (Node *) ctas;
 				}
 		;
@@ -4738,9 +4741,14 @@ create_mv_target:
 					$$->tableSpaceName = $5;
 					$$->viewQuery = NULL;		/* filled at analysis time */
 					$$->skipData = false;		/* might get changed later */
+					$$->ivm = false;
 				}
 		;
 
+incremental:	INCREMENTAL				{ $$ = true; }
+				| /*EMPTY*/				{ $$ = false; }
+		;
+
 OptNoLog:	UNLOGGED					{ $$ = RELPERSISTENCE_UNLOGGED; }
 			| /*EMPTY*/					{ $$ = RELPERSISTENCE_PERMANENT; }
 		;
@@ -17252,6 +17260,7 @@ unreserved_keyword:
 			| INCLUDE
 			| INCLUDING
 			| INCREMENT
+			| INCREMENTAL
 			| INDENT
 			| INDEX
 			| INDEXES
@@ -17820,6 +17829,7 @@ bare_label_keyword:
 			| INCLUDE
 			| INCLUDING
 			| INCREMENT
+			| INCREMENTAL
 			| INDENT
 			| INDEX
 			| INDEXES
diff --git a/src/include/nodes/primnodes.h b/src/include/nodes/primnodes.h
index 4a154606d2..5d61964f37 100644
--- a/src/include/nodes/primnodes.h
+++ b/src/include/nodes/primnodes.h
@@ -154,6 +154,7 @@ typedef struct IntoClause
 	/* materialized view's SELECT query */
 	Node	   *viewQuery pg_node_attr(query_jumble_ignore);
 	bool		skipData;		/* true for WITH NO DATA */
+	bool		ivm;			/* true for WITH IVM */
 } IntoClause;
 
 
diff --git a/src/include/parser/kwlist.h b/src/include/parser/kwlist.h
index 2331acac09..ab92b54642 100644
--- a/src/include/parser/kwlist.h
+++ b/src/include/parser/kwlist.h
@@ -207,6 +207,7 @@ PG_KEYWORD("in", IN_P, RESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("include", INCLUDE, UNRESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("including", INCLUDING, UNRESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("increment", INCREMENT, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("incremental", INCREMENTAL, UNRESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("indent", INDENT, UNRESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("index", INDEX, UNRESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("indexes", INDEXES, UNRESERVED_KEYWORD, BARE_LABEL)
-- 
2.25.1


--Multipart=_Mon__4_Mar_2024_11_58_46_+0900_UaponF/qQhQrVCFt
Content-Type: text/x-diff;
 name="v30-0002-Add-relisivm-column-to-pg_class-system-catalog.patch"
Content-Disposition: attachment;
 filename="v30-0002-Add-relisivm-column-to-pg_class-system-catalog.patch"
Content-Transfer-Encoding: 7bit



^ permalink  raw  reply  [nested|flat] 125+ messages in thread

* [PATCH v23 01/15] Add a syntax to create Incrementally Maintainable Materialized Views
@ 2019-12-20 01:05 Yugo Nagata <[email protected]>
  0 siblings, 0 replies; 125+ messages in thread

From: Yugo Nagata @ 2019-12-20 01:05 UTC (permalink / raw)

Allow to create Incrementally Maintainable Materialized View (IMMV)
by using INCREMENTAL option in CREATE MATERIALIZED VIEW command
as follow:

     CREATE [INCREMANTAL] MATERIALIZED VIEW xxxxx AS SELECT ....;
---
 src/backend/parser/gram.y     | 32 +++++++++++++++++++++-----------
 src/include/nodes/primnodes.h |  1 +
 src/include/parser/kwlist.h   |  1 +
 3 files changed, 23 insertions(+), 11 deletions(-)

diff --git a/src/backend/parser/gram.y b/src/backend/parser/gram.y
index 10da5c5c51..9c2382fed7 100644
--- a/src/backend/parser/gram.y
+++ b/src/backend/parser/gram.y
@@ -440,6 +440,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
 
 %type <range>	OptTempTableName
 %type <into>	into_clause create_as_target create_mv_target
+%type <boolean>	incremental
 
 %type <defelt>	createfunc_opt_item common_func_opt_item dostmt_opt_item
 %type <fun_param> func_arg func_arg_with_default table_func_column aggr_arg
@@ -669,7 +670,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
 	HANDLER HAVING HEADER_P HOLD HOUR_P
 
 	IDENTITY_P IF_P ILIKE IMMEDIATE IMMUTABLE IMPLICIT_P IMPORT_P IN_P INCLUDE
-	INCLUDING INCREMENT INDEX INDEXES INHERIT INHERITS INITIALLY INLINE_P
+	INCLUDING INCREMENT INCREMENTAL INDEX INDEXES INHERIT INHERITS INITIALLY INLINE_P
 	INNER_P INOUT INPUT_P INSENSITIVE INSERT INSTEAD INT_P INTEGER
 	INTERSECT INTERVAL INTO INVOKER IS ISNULL ISOLATION
 
@@ -4257,30 +4258,32 @@ opt_with_data:
  *****************************************************************************/
 
 CreateMatViewStmt:
-		CREATE OptNoLog MATERIALIZED VIEW create_mv_target AS SelectStmt opt_with_data
+		CREATE OptNoLog incremental MATERIALIZED VIEW create_mv_target AS SelectStmt opt_with_data
 				{
 					CreateTableAsStmt *ctas = makeNode(CreateTableAsStmt);
-					ctas->query = $7;
-					ctas->into = $5;
+					ctas->query = $8;
+					ctas->into = $6;
 					ctas->objtype = OBJECT_MATVIEW;
 					ctas->is_select_into = false;
 					ctas->if_not_exists = false;
 					/* cram additional flags into the IntoClause */
-					$5->rel->relpersistence = $2;
-					$5->skipData = !($8);
+					$6->rel->relpersistence = $2;
+					$6->skipData = !($9);
+					$6->ivm = $3;
 					$$ = (Node *) ctas;
 				}
-		| CREATE OptNoLog MATERIALIZED VIEW IF_P NOT EXISTS create_mv_target AS SelectStmt opt_with_data
+		| CREATE OptNoLog incremental MATERIALIZED VIEW IF_P NOT EXISTS create_mv_target AS SelectStmt opt_with_data
 				{
 					CreateTableAsStmt *ctas = makeNode(CreateTableAsStmt);
-					ctas->query = $10;
-					ctas->into = $8;
+					ctas->query = $11;
+					ctas->into = $9;
 					ctas->objtype = OBJECT_MATVIEW;
 					ctas->is_select_into = false;
 					ctas->if_not_exists = true;
 					/* cram additional flags into the IntoClause */
-					$8->rel->relpersistence = $2;
-					$8->skipData = !($11);
+					$9->rel->relpersistence = $2;
+					$9->skipData = !($12);
+					$9->ivm = $3;
 					$$ = (Node *) ctas;
 				}
 		;
@@ -4297,9 +4300,14 @@ create_mv_target:
 					$$->tableSpaceName = $5;
 					$$->viewQuery = NULL;		/* filled at analysis time */
 					$$->skipData = false;		/* might get changed later */
+					$$->ivm = false;
 				}
 		;
 
+incremental:	INCREMENTAL				{ $$ = true; }
+				| /*EMPTY*/				{ $$ = false; }
+		;
+
 OptNoLog:	UNLOGGED					{ $$ = RELPERSISTENCE_UNLOGGED; }
 			| /*EMPTY*/					{ $$ = RELPERSISTENCE_PERMANENT; }
 		;
@@ -15584,6 +15592,7 @@ unreserved_keyword:
 			| INCLUDE
 			| INCLUDING
 			| INCREMENT
+			| INCREMENTAL
 			| INDEX
 			| INDEXES
 			| INHERIT
@@ -16136,6 +16145,7 @@ bare_label_keyword:
 			| INCLUDE
 			| INCLUDING
 			| INCREMENT
+			| INCREMENTAL
 			| INDEX
 			| INDEXES
 			| INHERIT
diff --git a/src/include/nodes/primnodes.h b/src/include/nodes/primnodes.h
index c04282f91f..a0af94eb49 100644
--- a/src/include/nodes/primnodes.h
+++ b/src/include/nodes/primnodes.h
@@ -117,6 +117,7 @@ typedef struct IntoClause
 	char	   *tableSpaceName; /* table space to use, or NULL */
 	Node	   *viewQuery;		/* materialized view's SELECT query */
 	bool		skipData;		/* true for WITH NO DATA */
+	bool		ivm;			/* true for WITH IVM */
 } IntoClause;
 
 
diff --git a/src/include/parser/kwlist.h b/src/include/parser/kwlist.h
index f836acf876..2cafb4e7fe 100644
--- a/src/include/parser/kwlist.h
+++ b/src/include/parser/kwlist.h
@@ -205,6 +205,7 @@ PG_KEYWORD("in", IN_P, RESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("include", INCLUDE, UNRESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("including", INCLUDING, UNRESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("increment", INCREMENT, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("incremental", INCREMENTAL, UNRESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("index", INDEX, UNRESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("indexes", INDEXES, UNRESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("inherit", INHERIT, UNRESERVED_KEYWORD, BARE_LABEL)
-- 
2.17.1


--Multipart=_Mon__2_Aug_2021_15_28_34_+0900_wlHCjIpnD/FrGAKu
Content-Type: text/x-diff;
 name="v23-0002-Add-relisivm-column-to-pg_class-system-catalog.patch"
Content-Disposition: attachment;
 filename="v23-0002-Add-relisivm-column-to-pg_class-system-catalog.patch"
Content-Transfer-Encoding: 7bit



^ permalink  raw  reply  [nested|flat] 125+ messages in thread

* [PATCH v24 01/15] Add a syntax to create Incrementally Maintainable Materialized Views
@ 2019-12-20 01:05 Yugo Nagata <[email protected]>
  0 siblings, 0 replies; 125+ messages in thread

From: Yugo Nagata @ 2019-12-20 01:05 UTC (permalink / raw)

Allow to create Incrementally Maintainable Materialized View (IMMV)
by using INCREMENTAL option in CREATE MATERIALIZED VIEW command
as follow:

     CREATE [INCREMANTAL] MATERIALIZED VIEW xxxxx AS SELECT ....;
---
 src/backend/nodes/copyfuncs.c  |  1 +
 src/backend/nodes/equalfuncs.c |  1 +
 src/backend/nodes/outfuncs.c   |  1 +
 src/backend/nodes/readfuncs.c  |  1 +
 src/backend/parser/gram.y      | 32 +++++++++++++++++++++-----------
 src/include/nodes/primnodes.h  |  1 +
 src/include/parser/kwlist.h    |  1 +
 7 files changed, 27 insertions(+), 11 deletions(-)

diff --git a/src/backend/nodes/copyfuncs.c b/src/backend/nodes/copyfuncs.c
index 228387eaee..17be377aa7 100644
--- a/src/backend/nodes/copyfuncs.c
+++ b/src/backend/nodes/copyfuncs.c
@@ -1423,6 +1423,7 @@ _copyIntoClause(const IntoClause *from)
 	COPY_STRING_FIELD(tableSpaceName);
 	COPY_NODE_FIELD(viewQuery);
 	COPY_SCALAR_FIELD(skipData);
+	COPY_SCALAR_FIELD(ivm);
 
 	return newnode;
 }
diff --git a/src/backend/nodes/equalfuncs.c b/src/backend/nodes/equalfuncs.c
index 800f588b5c..ed74a5022c 100644
--- a/src/backend/nodes/equalfuncs.c
+++ b/src/backend/nodes/equalfuncs.c
@@ -155,6 +155,7 @@ _equalIntoClause(const IntoClause *a, const IntoClause *b)
 	COMPARE_STRING_FIELD(tableSpaceName);
 	COMPARE_NODE_FIELD(viewQuery);
 	COMPARE_SCALAR_FIELD(skipData);
+	COMPARE_SCALAR_FIELD(ivm);
 
 	return true;
 }
diff --git a/src/backend/nodes/outfuncs.c b/src/backend/nodes/outfuncs.c
index 2e5ed77e18..f6166f7859 100644
--- a/src/backend/nodes/outfuncs.c
+++ b/src/backend/nodes/outfuncs.c
@@ -1116,6 +1116,7 @@ _outIntoClause(StringInfo str, const IntoClause *node)
 	WRITE_STRING_FIELD(tableSpaceName);
 	WRITE_NODE_FIELD(viewQuery);
 	WRITE_BOOL_FIELD(skipData);
+	WRITE_BOOL_FIELD(ivm);
 }
 
 static void
diff --git a/src/backend/nodes/readfuncs.c b/src/backend/nodes/readfuncs.c
index abf08b7a2f..1e55a58f69 100644
--- a/src/backend/nodes/readfuncs.c
+++ b/src/backend/nodes/readfuncs.c
@@ -565,6 +565,7 @@ _readIntoClause(void)
 	READ_STRING_FIELD(tableSpaceName);
 	READ_NODE_FIELD(viewQuery);
 	READ_BOOL_FIELD(skipData);
+	READ_BOOL_FIELD(ivm);
 
 	READ_DONE();
 }
diff --git a/src/backend/parser/gram.y b/src/backend/parser/gram.y
index e3068a374e..46a812e550 100644
--- a/src/backend/parser/gram.y
+++ b/src/backend/parser/gram.y
@@ -439,6 +439,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
 
 %type <range>	OptTempTableName
 %type <into>	into_clause create_as_target create_mv_target
+%type <boolean>	incremental
 
 %type <defelt>	createfunc_opt_item common_func_opt_item dostmt_opt_item
 %type <fun_param> func_arg func_arg_with_default table_func_column aggr_arg
@@ -668,7 +669,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
 	HANDLER HAVING HEADER_P HOLD HOUR_P
 
 	IDENTITY_P IF_P ILIKE IMMEDIATE IMMUTABLE IMPLICIT_P IMPORT_P IN_P INCLUDE
-	INCLUDING INCREMENT INDEX INDEXES INHERIT INHERITS INITIALLY INLINE_P
+	INCLUDING INCREMENT INCREMENTAL INDEX INDEXES INHERIT INHERITS INITIALLY INLINE_P
 	INNER_P INOUT INPUT_P INSENSITIVE INSERT INSTEAD INT_P INTEGER
 	INTERSECT INTERVAL INTO INVOKER IS ISNULL ISOLATION
 
@@ -4264,30 +4265,32 @@ opt_with_data:
  *****************************************************************************/
 
 CreateMatViewStmt:
-		CREATE OptNoLog MATERIALIZED VIEW create_mv_target AS SelectStmt opt_with_data
+		CREATE OptNoLog incremental MATERIALIZED VIEW create_mv_target AS SelectStmt opt_with_data
 				{
 					CreateTableAsStmt *ctas = makeNode(CreateTableAsStmt);
-					ctas->query = $7;
-					ctas->into = $5;
+					ctas->query = $8;
+					ctas->into = $6;
 					ctas->objtype = OBJECT_MATVIEW;
 					ctas->is_select_into = false;
 					ctas->if_not_exists = false;
 					/* cram additional flags into the IntoClause */
-					$5->rel->relpersistence = $2;
-					$5->skipData = !($8);
+					$6->rel->relpersistence = $2;
+					$6->skipData = !($9);
+					$6->ivm = $3;
 					$$ = (Node *) ctas;
 				}
-		| CREATE OptNoLog MATERIALIZED VIEW IF_P NOT EXISTS create_mv_target AS SelectStmt opt_with_data
+		| CREATE OptNoLog incremental MATERIALIZED VIEW IF_P NOT EXISTS create_mv_target AS SelectStmt opt_with_data
 				{
 					CreateTableAsStmt *ctas = makeNode(CreateTableAsStmt);
-					ctas->query = $10;
-					ctas->into = $8;
+					ctas->query = $11;
+					ctas->into = $9;
 					ctas->objtype = OBJECT_MATVIEW;
 					ctas->is_select_into = false;
 					ctas->if_not_exists = true;
 					/* cram additional flags into the IntoClause */
-					$8->rel->relpersistence = $2;
-					$8->skipData = !($11);
+					$9->rel->relpersistence = $2;
+					$9->skipData = !($12);
+					$9->ivm = $3;
 					$$ = (Node *) ctas;
 				}
 		;
@@ -4304,9 +4307,14 @@ create_mv_target:
 					$$->tableSpaceName = $5;
 					$$->viewQuery = NULL;		/* filled at analysis time */
 					$$->skipData = false;		/* might get changed later */
+					$$->ivm = false;
 				}
 		;
 
+incremental:	INCREMENTAL				{ $$ = true; }
+				| /*EMPTY*/				{ $$ = false; }
+		;
+
 OptNoLog:	UNLOGGED					{ $$ = RELPERSISTENCE_UNLOGGED; }
 			| /*EMPTY*/					{ $$ = RELPERSISTENCE_PERMANENT; }
 		;
@@ -15606,6 +15614,7 @@ unreserved_keyword:
 			| INCLUDE
 			| INCLUDING
 			| INCREMENT
+			| INCREMENTAL
 			| INDEX
 			| INDEXES
 			| INHERIT
@@ -16158,6 +16167,7 @@ bare_label_keyword:
 			| INCLUDE
 			| INCLUDING
 			| INCREMENT
+			| INCREMENTAL
 			| INDEX
 			| INDEXES
 			| INHERIT
diff --git a/src/include/nodes/primnodes.h b/src/include/nodes/primnodes.h
index 433437643e..7ea80c7ded 100644
--- a/src/include/nodes/primnodes.h
+++ b/src/include/nodes/primnodes.h
@@ -117,6 +117,7 @@ typedef struct IntoClause
 	char	   *tableSpaceName; /* table space to use, or NULL */
 	Node	   *viewQuery;		/* materialized view's SELECT query */
 	bool		skipData;		/* true for WITH NO DATA */
+	bool		ivm;			/* true for WITH IVM */
 } IntoClause;
 
 
diff --git a/src/include/parser/kwlist.h b/src/include/parser/kwlist.h
index f836acf876..2cafb4e7fe 100644
--- a/src/include/parser/kwlist.h
+++ b/src/include/parser/kwlist.h
@@ -205,6 +205,7 @@ PG_KEYWORD("in", IN_P, RESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("include", INCLUDE, UNRESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("including", INCLUDING, UNRESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("increment", INCREMENT, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("incremental", INCREMENTAL, UNRESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("index", INDEX, UNRESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("indexes", INDEXES, UNRESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("inherit", INHERIT, UNRESERVED_KEYWORD, BARE_LABEL)
-- 
2.17.1


--Multipart=_Thu__23_Sep_2021_04_57_30_+0900_b5pmgR1N8oMaMz.U
Content-Type: text/x-diff;
 name="v24-0002-Add-relisivm-column-to-pg_class-system-catalog.patch"
Content-Disposition: attachment;
 filename="v24-0002-Add-relisivm-column-to-pg_class-system-catalog.patch"
Content-Transfer-Encoding: 7bit



^ permalink  raw  reply  [nested|flat] 125+ messages in thread

* [PATCH v39 1/8] Add a syntax to create Incrementally Maintainable Materialized Views
@ 2019-12-20 01:05 Yugo Nagata <[email protected]>
  0 siblings, 0 replies; 125+ messages in thread

From: Yugo Nagata @ 2019-12-20 01:05 UTC (permalink / raw)

Allow to create Incrementally Maintainable Materialized View (IMMV)
by using INCREMENTAL option in CREATE MATERIALIZED VIEW command
as follow:

     CREATE [INCREMANTAL] MATERIALIZED VIEW xxxxx AS SELECT ....;
---
 src/backend/parser/gram.y     | 32 +++++++++++++++++++++-----------
 src/include/nodes/primnodes.h |  1 +
 src/include/parser/kwlist.h   |  1 +
 3 files changed, 23 insertions(+), 11 deletions(-)

diff --git a/src/backend/parser/gram.y b/src/backend/parser/gram.y
index ff4e1388c55..9a353550c37 100644
--- a/src/backend/parser/gram.y
+++ b/src/backend/parser/gram.y
@@ -473,6 +473,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
 
 %type <range>	OptTempTableName
 %type <into>	into_clause create_as_target create_mv_target
+%type <boolean>	incremental
 
 %type <defelt>	createfunc_opt_item common_func_opt_item dostmt_opt_item
 %type <fun_param> func_arg func_arg_with_default table_func_column aggr_arg
@@ -776,7 +777,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
 	HANDLER HAVING HEADER_P HOLD HOUR_P
 
 	IDENTITY_P IF_P IGNORE_P ILIKE IMMEDIATE IMMUTABLE IMPLICIT_P IMPORT_P IN_P INCLUDE
-	INCLUDING INCREMENT INDENT INDEX INDEXES INHERIT INHERITS INITIALLY INLINE_P
+	INCLUDING INCREMENT INCREMENTAL INDENT INDEX INDEXES INHERIT INHERITS INITIALLY INLINE_P
 	INNER_P INOUT INPUT_P INSENSITIVE INSERT INSTEAD INT_P INTEGER
 	INTERSECT INTERVAL INTO INVOKER IS ISNULL ISOLATION
 
@@ -5039,32 +5040,34 @@ opt_with_data:
  *****************************************************************************/
 
 CreateMatViewStmt:
-		CREATE OptNoLog MATERIALIZED VIEW create_mv_target AS SelectStmt opt_with_data
+		CREATE OptNoLog incremental MATERIALIZED VIEW create_mv_target AS SelectStmt opt_with_data
 				{
 					CreateTableAsStmt *ctas = makeNode(CreateTableAsStmt);
 
-					ctas->query = $7;
-					ctas->into = $5;
+					ctas->query = $8;
+					ctas->into = $6;
 					ctas->objtype = OBJECT_MATVIEW;
 					ctas->is_select_into = false;
 					ctas->if_not_exists = false;
 					/* cram additional flags into the IntoClause */
-					$5->rel->relpersistence = $2;
-					$5->skipData = !($8);
+					$6->rel->relpersistence = $2;
+					$6->skipData = !($9);
+					$6->ivm = $3;
 					$$ = (Node *) ctas;
 				}
-		| CREATE OptNoLog MATERIALIZED VIEW IF_P NOT EXISTS create_mv_target AS SelectStmt opt_with_data
+		| CREATE OptNoLog incremental MATERIALIZED VIEW IF_P NOT EXISTS create_mv_target AS SelectStmt opt_with_data
 				{
 					CreateTableAsStmt *ctas = makeNode(CreateTableAsStmt);
 
-					ctas->query = $10;
-					ctas->into = $8;
+					ctas->query = $11;
+					ctas->into = $9;
 					ctas->objtype = OBJECT_MATVIEW;
 					ctas->is_select_into = false;
 					ctas->if_not_exists = true;
 					/* cram additional flags into the IntoClause */
-					$8->rel->relpersistence = $2;
-					$8->skipData = !($11);
+					$9->rel->relpersistence = $2;
+					$9->skipData = !($12);
+					$9->ivm = $3;
 					$$ = (Node *) ctas;
 				}
 		;
@@ -5081,9 +5084,14 @@ create_mv_target:
 					$$->tableSpaceName = $5;
 					$$->viewQuery = NULL;		/* filled at analysis time */
 					$$->skipData = false;		/* might get changed later */
+					$$->ivm = false;
 				}
 		;
 
+incremental:	INCREMENTAL				{ $$ = true; }
+				| /*EMPTY*/				{ $$ = false; }
+		;
+
 OptNoLog:	UNLOGGED					{ $$ = RELPERSISTENCE_UNLOGGED; }
 			| /*EMPTY*/					{ $$ = RELPERSISTENCE_PERMANENT; }
 		;
@@ -18948,6 +18956,7 @@ unreserved_keyword:
 			| INCLUDE
 			| INCLUDING
 			| INCREMENT
+			| INCREMENTAL
 			| INDENT
 			| INDEX
 			| INDEXES
@@ -19554,6 +19563,7 @@ bare_label_keyword:
 			| INCLUDE
 			| INCLUDING
 			| INCREMENT
+			| INCREMENTAL
 			| INDENT
 			| INDEX
 			| INDEXES
diff --git a/src/include/nodes/primnodes.h b/src/include/nodes/primnodes.h
index bb05aeebee4..da35b15cc27 100644
--- a/src/include/nodes/primnodes.h
+++ b/src/include/nodes/primnodes.h
@@ -170,6 +170,7 @@ typedef struct IntoClause
 	/* materialized view's SELECT query */
 	struct Query *viewQuery pg_node_attr(query_jumble_ignore);
 	bool		skipData;		/* true for WITH NO DATA */
+	bool		ivm;			/* true for WITH IVM */
 } IntoClause;
 
 
diff --git a/src/include/parser/kwlist.h b/src/include/parser/kwlist.h
index 51ead54f015..7849ee7f960 100644
--- a/src/include/parser/kwlist.h
+++ b/src/include/parser/kwlist.h
@@ -216,6 +216,7 @@ PG_KEYWORD("in", IN_P, RESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("include", INCLUDE, UNRESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("including", INCLUDING, UNRESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("increment", INCREMENT, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("incremental", INCREMENTAL, UNRESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("indent", INDENT, UNRESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("index", INDEX, UNRESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("indexes", INDEXES, UNRESERVED_KEYWORD, BARE_LABEL)
-- 
2.43.0


--Multipart=_Fri__3_Jul_2026_19_11_16_+0900_CskxSIu_iGcDMEyM--






^ permalink  raw  reply  [nested|flat] 125+ messages in thread

* [PATCH v37 01/11] Add a syntax to create Incrementally Maintainable Materialized Views
@ 2019-12-20 01:05 Yugo Nagata <[email protected]>
  0 siblings, 0 replies; 125+ messages in thread

From: Yugo Nagata @ 2019-12-20 01:05 UTC (permalink / raw)

Allow to create Incrementally Maintainable Materialized View (IMMV)
by using INCREMENTAL option in CREATE MATERIALIZED VIEW command
as follow:

     CREATE [INCREMANTAL] MATERIALIZED VIEW xxxxx AS SELECT ....;
---
 src/backend/parser/gram.y     | 32 +++++++++++++++++++++-----------
 src/include/nodes/primnodes.h |  1 +
 src/include/parser/kwlist.h   |  1 +
 3 files changed, 23 insertions(+), 11 deletions(-)

diff --git a/src/backend/parser/gram.y b/src/backend/parser/gram.y
index ff4e1388c55..9a353550c37 100644
--- a/src/backend/parser/gram.y
+++ b/src/backend/parser/gram.y
@@ -473,6 +473,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
 
 %type <range>	OptTempTableName
 %type <into>	into_clause create_as_target create_mv_target
+%type <boolean>	incremental
 
 %type <defelt>	createfunc_opt_item common_func_opt_item dostmt_opt_item
 %type <fun_param> func_arg func_arg_with_default table_func_column aggr_arg
@@ -776,7 +777,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
 	HANDLER HAVING HEADER_P HOLD HOUR_P
 
 	IDENTITY_P IF_P IGNORE_P ILIKE IMMEDIATE IMMUTABLE IMPLICIT_P IMPORT_P IN_P INCLUDE
-	INCLUDING INCREMENT INDENT INDEX INDEXES INHERIT INHERITS INITIALLY INLINE_P
+	INCLUDING INCREMENT INCREMENTAL INDENT INDEX INDEXES INHERIT INHERITS INITIALLY INLINE_P
 	INNER_P INOUT INPUT_P INSENSITIVE INSERT INSTEAD INT_P INTEGER
 	INTERSECT INTERVAL INTO INVOKER IS ISNULL ISOLATION
 
@@ -5039,32 +5040,34 @@ opt_with_data:
  *****************************************************************************/
 
 CreateMatViewStmt:
-		CREATE OptNoLog MATERIALIZED VIEW create_mv_target AS SelectStmt opt_with_data
+		CREATE OptNoLog incremental MATERIALIZED VIEW create_mv_target AS SelectStmt opt_with_data
 				{
 					CreateTableAsStmt *ctas = makeNode(CreateTableAsStmt);
 
-					ctas->query = $7;
-					ctas->into = $5;
+					ctas->query = $8;
+					ctas->into = $6;
 					ctas->objtype = OBJECT_MATVIEW;
 					ctas->is_select_into = false;
 					ctas->if_not_exists = false;
 					/* cram additional flags into the IntoClause */
-					$5->rel->relpersistence = $2;
-					$5->skipData = !($8);
+					$6->rel->relpersistence = $2;
+					$6->skipData = !($9);
+					$6->ivm = $3;
 					$$ = (Node *) ctas;
 				}
-		| CREATE OptNoLog MATERIALIZED VIEW IF_P NOT EXISTS create_mv_target AS SelectStmt opt_with_data
+		| CREATE OptNoLog incremental MATERIALIZED VIEW IF_P NOT EXISTS create_mv_target AS SelectStmt opt_with_data
 				{
 					CreateTableAsStmt *ctas = makeNode(CreateTableAsStmt);
 
-					ctas->query = $10;
-					ctas->into = $8;
+					ctas->query = $11;
+					ctas->into = $9;
 					ctas->objtype = OBJECT_MATVIEW;
 					ctas->is_select_into = false;
 					ctas->if_not_exists = true;
 					/* cram additional flags into the IntoClause */
-					$8->rel->relpersistence = $2;
-					$8->skipData = !($11);
+					$9->rel->relpersistence = $2;
+					$9->skipData = !($12);
+					$9->ivm = $3;
 					$$ = (Node *) ctas;
 				}
 		;
@@ -5081,9 +5084,14 @@ create_mv_target:
 					$$->tableSpaceName = $5;
 					$$->viewQuery = NULL;		/* filled at analysis time */
 					$$->skipData = false;		/* might get changed later */
+					$$->ivm = false;
 				}
 		;
 
+incremental:	INCREMENTAL				{ $$ = true; }
+				| /*EMPTY*/				{ $$ = false; }
+		;
+
 OptNoLog:	UNLOGGED					{ $$ = RELPERSISTENCE_UNLOGGED; }
 			| /*EMPTY*/					{ $$ = RELPERSISTENCE_PERMANENT; }
 		;
@@ -18948,6 +18956,7 @@ unreserved_keyword:
 			| INCLUDE
 			| INCLUDING
 			| INCREMENT
+			| INCREMENTAL
 			| INDENT
 			| INDEX
 			| INDEXES
@@ -19554,6 +19563,7 @@ bare_label_keyword:
 			| INCLUDE
 			| INCLUDING
 			| INCREMENT
+			| INCREMENTAL
 			| INDENT
 			| INDEX
 			| INDEXES
diff --git a/src/include/nodes/primnodes.h b/src/include/nodes/primnodes.h
index 7977ee24783..213adeec2e2 100644
--- a/src/include/nodes/primnodes.h
+++ b/src/include/nodes/primnodes.h
@@ -170,6 +170,7 @@ typedef struct IntoClause
 	/* materialized view's SELECT query */
 	struct Query *viewQuery pg_node_attr(query_jumble_ignore);
 	bool		skipData;		/* true for WITH NO DATA */
+	bool		ivm;			/* true for WITH IVM */
 } IntoClause;
 
 
diff --git a/src/include/parser/kwlist.h b/src/include/parser/kwlist.h
index 51ead54f015..7849ee7f960 100644
--- a/src/include/parser/kwlist.h
+++ b/src/include/parser/kwlist.h
@@ -216,6 +216,7 @@ PG_KEYWORD("in", IN_P, RESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("include", INCLUDE, UNRESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("including", INCLUDING, UNRESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("increment", INCREMENT, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("incremental", INCREMENTAL, UNRESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("indent", INDENT, UNRESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("index", INDEX, UNRESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("indexes", INDEXES, UNRESERVED_KEYWORD, BARE_LABEL)
-- 
2.43.0


--Multipart=_Fri__29_May_2026_23_14_17_+0900_Te0o73X2VqYK57Gd--





^ permalink  raw  reply  [nested|flat] 125+ messages in thread

* [PATCH v32 01/11] Add a syntax to create Incrementally Maintainable Materialized Views
@ 2019-12-20 01:05 Yugo Nagata <[email protected]>
  0 siblings, 0 replies; 125+ messages in thread

From: Yugo Nagata @ 2019-12-20 01:05 UTC (permalink / raw)

Allow to create Incrementally Maintainable Materialized View (IMMV)
by using INCREMENTAL option in CREATE MATERIALIZED VIEW command
as follow:

     CREATE [INCREMANTAL] MATERIALIZED VIEW xxxxx AS SELECT ....;
---
 src/backend/parser/gram.y     | 32 +++++++++++++++++++++-----------
 src/include/nodes/primnodes.h |  1 +
 src/include/parser/kwlist.h   |  1 +
 3 files changed, 23 insertions(+), 11 deletions(-)

diff --git a/src/backend/parser/gram.y b/src/backend/parser/gram.y
index 682748eb4b..91df005a19 100644
--- a/src/backend/parser/gram.y
+++ b/src/backend/parser/gram.y
@@ -468,6 +468,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
 
 %type <range>	OptTempTableName
 %type <into>	into_clause create_as_target create_mv_target
+%type <boolean>	incremental
 
 %type <defelt>	createfunc_opt_item common_func_opt_item dostmt_opt_item
 %type <fun_param> func_arg func_arg_with_default table_func_column aggr_arg
@@ -732,7 +733,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
 	HANDLER HAVING HEADER_P HOLD HOUR_P
 
 	IDENTITY_P IF_P ILIKE IMMEDIATE IMMUTABLE IMPLICIT_P IMPORT_P IN_P INCLUDE
-	INCLUDING INCREMENT INDENT INDEX INDEXES INHERIT INHERITS INITIALLY INLINE_P
+	INCLUDING INCREMENT INCREMENTAL INDENT INDEX INDEXES INHERIT INHERITS INITIALLY INLINE_P
 	INNER_P INOUT INPUT_P INSENSITIVE INSERT INSTEAD INT_P INTEGER
 	INTERSECT INTERVAL INTO INVOKER IS ISNULL ISOLATION
 
@@ -4734,32 +4735,34 @@ opt_with_data:
  *****************************************************************************/
 
 CreateMatViewStmt:
-		CREATE OptNoLog MATERIALIZED VIEW create_mv_target AS SelectStmt opt_with_data
+		CREATE OptNoLog incremental MATERIALIZED VIEW create_mv_target AS SelectStmt opt_with_data
 				{
 					CreateTableAsStmt *ctas = makeNode(CreateTableAsStmt);
 
-					ctas->query = $7;
-					ctas->into = $5;
+					ctas->query = $8;
+					ctas->into = $6;
 					ctas->objtype = OBJECT_MATVIEW;
 					ctas->is_select_into = false;
 					ctas->if_not_exists = false;
 					/* cram additional flags into the IntoClause */
-					$5->rel->relpersistence = $2;
-					$5->skipData = !($8);
+					$6->rel->relpersistence = $2;
+					$6->skipData = !($9);
+					$6->ivm = $3;
 					$$ = (Node *) ctas;
 				}
-		| CREATE OptNoLog MATERIALIZED VIEW IF_P NOT EXISTS create_mv_target AS SelectStmt opt_with_data
+		| CREATE OptNoLog incremental MATERIALIZED VIEW IF_P NOT EXISTS create_mv_target AS SelectStmt opt_with_data
 				{
 					CreateTableAsStmt *ctas = makeNode(CreateTableAsStmt);
 
-					ctas->query = $10;
-					ctas->into = $8;
+					ctas->query = $11;
+					ctas->into = $9;
 					ctas->objtype = OBJECT_MATVIEW;
 					ctas->is_select_into = false;
 					ctas->if_not_exists = true;
 					/* cram additional flags into the IntoClause */
-					$8->rel->relpersistence = $2;
-					$8->skipData = !($11);
+					$9->rel->relpersistence = $2;
+					$9->skipData = !($12);
+					$9->ivm = $3;
 					$$ = (Node *) ctas;
 				}
 		;
@@ -4776,9 +4779,14 @@ create_mv_target:
 					$$->tableSpaceName = $5;
 					$$->viewQuery = NULL;		/* filled at analysis time */
 					$$->skipData = false;		/* might get changed later */
+					$$->ivm = false;
 				}
 		;
 
+incremental:	INCREMENTAL				{ $$ = true; }
+				| /*EMPTY*/				{ $$ = false; }
+		;
+
 OptNoLog:	UNLOGGED					{ $$ = RELPERSISTENCE_UNLOGGED; }
 			| /*EMPTY*/					{ $$ = RELPERSISTENCE_PERMANENT; }
 		;
@@ -17454,6 +17462,7 @@ unreserved_keyword:
 			| INCLUDE
 			| INCLUDING
 			| INCREMENT
+			| INCREMENTAL
 			| INDENT
 			| INDEX
 			| INDEXES
@@ -18037,6 +18046,7 @@ bare_label_keyword:
 			| INCLUDE
 			| INCLUDING
 			| INCREMENT
+			| INCREMENTAL
 			| INDENT
 			| INDEX
 			| INDEXES
diff --git a/src/include/nodes/primnodes.h b/src/include/nodes/primnodes.h
index aa727e722c..9d0ba29a2a 100644
--- a/src/include/nodes/primnodes.h
+++ b/src/include/nodes/primnodes.h
@@ -154,6 +154,7 @@ typedef struct IntoClause
 	/* materialized view's SELECT query */
 	Node	   *viewQuery pg_node_attr(query_jumble_ignore);
 	bool		skipData;		/* true for WITH NO DATA */
+	bool		ivm;			/* true for WITH IVM */
 } IntoClause;
 
 
diff --git a/src/include/parser/kwlist.h b/src/include/parser/kwlist.h
index 6c959e85d5..94c4a3de1a 100644
--- a/src/include/parser/kwlist.h
+++ b/src/include/parser/kwlist.h
@@ -210,6 +210,7 @@ PG_KEYWORD("in", IN_P, RESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("include", INCLUDE, UNRESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("including", INCLUDING, UNRESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("increment", INCREMENT, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("incremental", INCREMENTAL, UNRESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("indent", INDENT, UNRESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("index", INDEX, UNRESERVED_KEYWORD, BARE_LABEL)
 PG_KEYWORD("indexes", INDEXES, UNRESERVED_KEYWORD, BARE_LABEL)
-- 
2.25.1


--Multipart=_Sun__31_Mar_2024_22_59_31_+0900_msknEviJj08_wgqO
Content-Type: text/x-diff;
 name="v32-0002-Add-relisivm-column-to-pg_class-system-catalog.patch"
Content-Disposition: attachment;
 filename="v32-0002-Add-relisivm-column-to-pg_class-system-catalog.patch"
Content-Transfer-Encoding: 7bit



^ permalink  raw  reply  [nested|flat] 125+ messages in thread

* ResourceOwner refactoring
@ 2020-11-17 14:21 Heikki Linnakangas <[email protected]>
  2020-11-18 08:06 ` Re: ResourceOwner refactoring Michael Paquier <[email protected]>
  2020-11-19 10:40 ` Re: ResourceOwner refactoring Craig Ringer <[email protected]>
  0 siblings, 2 replies; 125+ messages in thread

From: Heikki Linnakangas @ 2020-11-17 14:21 UTC (permalink / raw)
  To: pgsql-hackers; +Cc: Michael Paquier <[email protected]>; Kyotaro Horiguchi <[email protected]>

Two recent patches that I have reviewed have reminded me that the 
ResourceOwner interface is a bit clunky. There are two issues:

1. Performance. It's fast enough in most use, but when I was testing 
Kyotaro's catcache patches [1], the Resowner functions to track catcache 
references nevertheless showed up, accounting for about 20% of the CPU 
time used by catcache lookups.

2. It's difficult for extensions to use. There is a callback mechanism 
for extensions, but it's much less convenient to use than the built-in 
functions. The pgcrypto code uses the callbacks currently, and Michael's 
patch [2] would move that support for tracking OpenSSL contexts to the 
core, which makes it a lot more convenient for pgcrypto. Wouldn't it be 
nice if extensions could have the same ergonomics as built-in code, if 
they need to track external resources?

Attached patch refactors the ResourceOwner internals to do that.

The current code in HEAD has a separate array for each "kind" of object 
that can be tracked. The number of different kinds of objects has grown 
over the years, currently there is support for tracking buffers, files, 
catcache, relcache and plancache references, tupledescs, snapshots, DSM 
segments and LLVM JIT contexts. And locks, which are a bit special.

In the patch, I'm using the same array to hold all kinds of objects, and 
carry another field with each tracked object, to tell what kind of an 
object it is. An extension can define a new object kind, by defining 
Release and PrintLeakWarning callback functions for it. The code in 
resowner.c is now much more generic, as it doesn't need to know about 
all the different object kinds anymore (with a couple of exceptions). In 
the new scheme, there is one small fixed-size array to hold a few most 
recently-added references, that is linear-searched, and older entries 
are moved to a hash table.

I haven't done thorough performance testing of this, but with some quick 
testing with Kyotaro's "catcachebench" to stress-test syscache lookups, 
this performs a little better. In that test, all the activity happens in 
the small array portion, but I believe that's true for most use cases.

Thoughts? Can anyone suggest test scenarios to verify the performance of 
this?

[1] 
https://www.postgresql.org/message-id/20201106.172958.1103727352904717607.horikyota.ntt%40gmail.com

[2] https://www.postgresql.org/message-id/[email protected]

- Heikki


Attachments:

  [text/x-patch] v1-resowner-refactor.patch (76.2K, ../../[email protected]/2-v1-resowner-refactor.patch)
  download | inline diff:
diff --git a/src/backend/access/common/tupdesc.c b/src/backend/access/common/tupdesc.c
index 30c30cf3a2e..c099f04f551 100644
--- a/src/backend/access/common/tupdesc.c
+++ b/src/backend/access/common/tupdesc.c
@@ -29,9 +29,21 @@
 #include "utils/acl.h"
 #include "utils/builtins.h"
 #include "utils/datum.h"
-#include "utils/resowner_private.h"
+#include "utils/resowner.h"
 #include "utils/syscache.h"
 
+/* ResourceOwner callbacks to hold tupledesc references  */
+static void ResOwnerReleaseTupleDesc(Datum res);
+static void ResOwnerPrintTupleDescLeakWarning(Datum res);
+
+static ResourceOwnerFuncs tupdesc_resowner_funcs =
+{
+	/* relcache references */
+	.name = "tupdesc reference",
+	.phase = RESOURCE_RELEASE_AFTER_LOCKS,
+	.ReleaseResource = ResOwnerReleaseTupleDesc,
+	.PrintLeakWarning = ResOwnerPrintTupleDescLeakWarning
+};
 
 /*
  * CreateTemplateTupleDesc
@@ -376,9 +388,10 @@ IncrTupleDescRefCount(TupleDesc tupdesc)
 {
 	Assert(tupdesc->tdrefcount >= 0);
 
-	ResourceOwnerEnlargeTupleDescs(CurrentResourceOwner);
+	ResourceOwnerEnlarge(CurrentResourceOwner);
 	tupdesc->tdrefcount++;
-	ResourceOwnerRememberTupleDesc(CurrentResourceOwner, tupdesc);
+	ResourceOwnerRemember(CurrentResourceOwner, PointerGetDatum(tupdesc),
+						  &tupdesc_resowner_funcs);
 }
 
 /*
@@ -394,7 +407,8 @@ DecrTupleDescRefCount(TupleDesc tupdesc)
 {
 	Assert(tupdesc->tdrefcount > 0);
 
-	ResourceOwnerForgetTupleDesc(CurrentResourceOwner, tupdesc);
+	ResourceOwnerForget(CurrentResourceOwner, PointerGetDatum(tupdesc),
+						&tupdesc_resowner_funcs);
 	if (--tupdesc->tdrefcount == 0)
 		FreeTupleDesc(tupdesc);
 }
@@ -925,3 +939,23 @@ BuildDescFromLists(List *names, List *types, List *typmods, List *collations)
 
 	return desc;
 }
+
+
+/*
+ * ResourceOwner callbacks
+ */
+static void
+ResOwnerReleaseTupleDesc(Datum res)
+{
+	DecrTupleDescRefCount((TupleDesc) DatumGetPointer(res));
+}
+
+static void
+ResOwnerPrintTupleDescLeakWarning(Datum res)
+{
+	TupleDesc tupdesc = (TupleDesc) DatumGetPointer(res);
+
+	elog(WARNING,
+		 "TupleDesc reference leak: TupleDesc %p (%u,%d) still referenced",
+		 tupdesc, tupdesc->tdtypeid, tupdesc->tdtypmod);
+}
diff --git a/src/backend/jit/jit.c b/src/backend/jit/jit.c
index 5ca3f922fed..c44080b9742 100644
--- a/src/backend/jit/jit.c
+++ b/src/backend/jit/jit.c
@@ -26,7 +26,6 @@
 #include "jit/jit.h"
 #include "miscadmin.h"
 #include "utils/fmgrprotos.h"
-#include "utils/resowner_private.h"
 
 /* GUCs */
 bool		jit_enabled = true;
@@ -140,7 +139,6 @@ jit_release_context(JitContext *context)
 	if (provider_successfully_loaded)
 		provider.release_context(context);
 
-	ResourceOwnerForgetJIT(context->resowner, PointerGetDatum(context));
 	pfree(context);
 }
 
diff --git a/src/backend/jit/llvm/llvmjit.c b/src/backend/jit/llvm/llvmjit.c
index 40a439326c6..edea0388c04 100644
--- a/src/backend/jit/llvm/llvmjit.c
+++ b/src/backend/jit/llvm/llvmjit.c
@@ -40,7 +40,7 @@
 #include "portability/instr_time.h"
 #include "storage/ipc.h"
 #include "utils/memutils.h"
-#include "utils/resowner_private.h"
+#include "utils/resowner.h"
 
 /* Handle of a module emitted via ORC JIT */
 typedef struct LLVMJitHandle
@@ -121,8 +121,20 @@ static LLVMOrcLLJITRef llvm_create_jit_instance(LLVMTargetMachineRef tm);
 static char *llvm_error_message(LLVMErrorRef error);
 #endif							/* LLVM_VERSION_MAJOR > 11 */
 
-PG_MODULE_MAGIC;
+/* ResourceOwner callbacks to hold JitContexts  */
+static void ResOwnerReleaseJitContext(Datum res);
+static void ResOwnerPrintJitContextLeakWarning(Datum res);
+
+static ResourceOwnerFuncs jit_funcs =
+{
+	/* relcache references */
+	.name = "LLVM JIT context",
+	.phase = RESOURCE_RELEASE_BEFORE_LOCKS,
+	.ReleaseResource = ResOwnerReleaseJitContext,
+	.PrintLeakWarning = ResOwnerPrintJitContextLeakWarning
+};
 
+PG_MODULE_MAGIC;
 
 /*
  * Initialize LLVM JIT provider.
@@ -151,7 +163,7 @@ llvm_create_context(int jitFlags)
 
 	llvm_session_initialize();
 
-	ResourceOwnerEnlargeJIT(CurrentResourceOwner);
+	ResourceOwnerEnlarge(CurrentResourceOwner);
 
 	context = MemoryContextAllocZero(TopMemoryContext,
 									 sizeof(LLVMJitContext));
@@ -159,7 +171,7 @@ llvm_create_context(int jitFlags)
 
 	/* ensure cleanup */
 	context->base.resowner = CurrentResourceOwner;
-	ResourceOwnerRememberJIT(CurrentResourceOwner, PointerGetDatum(context));
+	ResourceOwnerRemember(CurrentResourceOwner, PointerGetDatum(context), &jit_funcs);
 
 	return context;
 }
@@ -221,6 +233,8 @@ llvm_release_context(JitContext *context)
 
 		pfree(jit_handle);
 	}
+
+	ResourceOwnerForget(context->resowner, PointerGetDatum(context), &jit_funcs);
 }
 
 /*
@@ -1210,3 +1224,21 @@ llvm_error_message(LLVMErrorRef error)
 }
 
 #endif							/* LLVM_VERSION_MAJOR > 11 */
+
+/*
+ * ResourceOwner callbacks
+ */
+static void
+ResOwnerReleaseJitContext(Datum res)
+{
+	jit_release_context((JitContext *) PointerGetDatum(res));
+}
+
+static void
+ResOwnerPrintJitContextLeakWarning(Datum res)
+{
+	/* XXX: We used to not print these. Was that intentional? */
+	JitContext *context = (JitContext *) PointerGetDatum(res);
+
+	elog(WARNING, "JIT context leak: context %p still referenced", context);
+}
diff --git a/src/backend/storage/buffer/bufmgr.c b/src/backend/storage/buffer/bufmgr.c
index ad0d1a9abc0..11a42cbdea6 100644
--- a/src/backend/storage/buffer/bufmgr.c
+++ b/src/backend/storage/buffer/bufmgr.c
@@ -52,7 +52,7 @@
 #include "utils/memdebug.h"
 #include "utils/ps_status.h"
 #include "utils/rel.h"
-#include "utils/resowner_private.h"
+#include "utils/resowner.h"
 #include "utils/timestamp.h"
 
 
@@ -198,6 +198,18 @@ static PrivateRefCountEntry *GetPrivateRefCountEntry(Buffer buffer, bool do_move
 static inline int32 GetPrivateRefCount(Buffer buffer);
 static void ForgetPrivateRefCountEntry(PrivateRefCountEntry *ref);
 
+/* ResourceOwner callbacks to hold buffer pins */
+static void ResOwnerReleaseBuffer(Datum res);
+static void ResOwnerPrintBufferLeakWarning(Datum res);
+
+ResourceOwnerFuncs buffer_resowner_funcs =
+{
+	.name = "buffer",
+	.phase = RESOURCE_RELEASE_BEFORE_LOCKS,
+	.ReleaseResource = ResOwnerReleaseBuffer,
+	.PrintLeakWarning = ResOwnerPrintBufferLeakWarning
+};
+
 /*
  * Ensure that the PrivateRefCountArray has sufficient space to store one more
  * entry. This has to be called before using NewPrivateRefCountEntry() to fill
@@ -727,7 +739,7 @@ ReadBuffer_common(SMgrRelation smgr, char relpersistence, ForkNumber forkNum,
 	*hit = false;
 
 	/* Make sure we will have room to remember the buffer pin */
-	ResourceOwnerEnlargeBuffers(CurrentResourceOwner);
+	ResourceOwnerEnlarge(CurrentResourceOwner);
 
 	isExtend = (blockNum == P_NEW);
 
@@ -1846,7 +1858,7 @@ BufferSync(int flags)
 	WritebackContext wb_context;
 
 	/* Make sure we can handle the pin inside SyncOneBuffer */
-	ResourceOwnerEnlargeBuffers(CurrentResourceOwner);
+	ResourceOwnerEnlarge(CurrentResourceOwner);
 
 	/*
 	 * Unless this is a shutdown checkpoint or we have been explicitly told,
@@ -2323,7 +2335,7 @@ BgBufferSync(WritebackContext *wb_context)
 	 */
 
 	/* Make sure we can handle the pin inside SyncOneBuffer */
-	ResourceOwnerEnlargeBuffers(CurrentResourceOwner);
+	ResourceOwnerEnlarge(CurrentResourceOwner);
 
 	num_to_scan = bufs_to_lap;
 	num_written = 0;
@@ -3301,7 +3313,7 @@ FlushRelationBuffers(Relation rel)
 	}
 
 	/* Make sure we can handle the pin inside the loop */
-	ResourceOwnerEnlargeBuffers(CurrentResourceOwner);
+	ResourceOwnerEnlarge(CurrentResourceOwner);
 
 	for (i = 0; i < NBuffers; i++)
 	{
@@ -3374,7 +3386,7 @@ FlushRelationsAllBuffers(SMgrRelation *smgrs, int nrels)
 		pg_qsort(srels, nrels, sizeof(SMgrSortArray), rnode_comparator);
 
 	/* Make sure we can handle the pin inside the loop */
-	ResourceOwnerEnlargeBuffers(CurrentResourceOwner);
+	ResourceOwnerEnlarge(CurrentResourceOwner);
 
 	for (i = 0; i < NBuffers; i++)
 	{
@@ -3453,7 +3465,7 @@ FlushDatabaseBuffers(Oid dbid)
 	BufferDesc *bufHdr;
 
 	/* Make sure we can handle the pin inside the loop */
-	ResourceOwnerEnlargeBuffers(CurrentResourceOwner);
+	ResourceOwnerEnlarge(CurrentResourceOwner);
 
 	for (i = 0; i < NBuffers; i++)
 	{
@@ -3551,7 +3563,7 @@ void
 IncrBufferRefCount(Buffer buffer)
 {
 	Assert(BufferIsPinned(buffer));
-	ResourceOwnerEnlargeBuffers(CurrentResourceOwner);
+	ResourceOwnerEnlarge(CurrentResourceOwner);
 	if (BufferIsLocal(buffer))
 		LocalRefCount[-buffer - 1]++;
 	else
@@ -4585,3 +4597,18 @@ TestForOldSnapshot_impl(Snapshot snapshot, Relation relation)
 				(errcode(ERRCODE_SNAPSHOT_TOO_OLD),
 				 errmsg("snapshot too old")));
 }
+
+/*
+ * ResourceOwner callbacks
+ */
+static void
+ResOwnerReleaseBuffer(Datum res)
+{
+	ReleaseBuffer(DatumGetInt32(res));
+}
+
+static void
+ResOwnerPrintBufferLeakWarning(Datum res)
+{
+	PrintBufferLeakWarning(DatumGetInt32(res));
+}
diff --git a/src/backend/storage/buffer/localbuf.c b/src/backend/storage/buffer/localbuf.c
index 6ffd7b33062..cbf9819cba5 100644
--- a/src/backend/storage/buffer/localbuf.c
+++ b/src/backend/storage/buffer/localbuf.c
@@ -22,7 +22,7 @@
 #include "storage/bufmgr.h"
 #include "utils/guc.h"
 #include "utils/memutils.h"
-#include "utils/resowner_private.h"
+#include "utils/resowner.h"
 
 
 /*#define LBDEBUG*/
diff --git a/src/backend/storage/file/fd.c b/src/backend/storage/file/fd.c
index 05abcf72d68..ba6262c83da 100644
--- a/src/backend/storage/file/fd.c
+++ b/src/backend/storage/file/fd.c
@@ -96,7 +96,7 @@
 #include "storage/fd.h"
 #include "storage/ipc.h"
 #include "utils/guc.h"
-#include "utils/resowner_private.h"
+#include "utils/resowner.h"
 
 /* Define PG_FLUSH_DATA_WORKS if we have an implementation for pg_flush_data */
 #if defined(HAVE_SYNC_FILE_RANGE)
@@ -339,6 +339,24 @@ static void unlink_if_exists_fname(const char *fname, bool isdir, int elevel);
 static int	fsync_parent_path(const char *fname, int elevel);
 
 
+/* ResourceOwner callbacks to hold virtual file descriptors */
+static void ResOwnerReleaseFile(Datum res);
+static void ResOwnerPrintFileLeakWarning(Datum res);
+
+static ResourceOwnerFuncs file_resowner_funcs =
+{
+	.name = "File",
+	.phase = RESOURCE_RELEASE_AFTER_LOCKS,
+	.ReleaseResource = ResOwnerReleaseFile,
+	.PrintLeakWarning = ResOwnerPrintFileLeakWarning
+};
+
+/* Convenience wrappers over ResourceOwnerRemember/Forget */
+#define ResourceOwnerRememberFile(owner, file) \
+	ResourceOwnerRemember(owner, Int32GetDatum(file), &file_resowner_funcs)
+#define ResourceOwnerForgetFile(owner, file) \
+	ResourceOwnerForget(owner, Int32GetDatum(file), &file_resowner_funcs)
+
 /*
  * pg_fsync --- do fsync with or without writethrough
  */
@@ -1402,7 +1420,7 @@ ReportTemporaryFileUsage(const char *path, off_t size)
 
 /*
  * Called to register a temporary file for automatic close.
- * ResourceOwnerEnlargeFiles(CurrentResourceOwner) must have been called
+ * ResourceOwnerEnlarge(CurrentResourceOwner) must have been called
  * before the file was opened.
  */
 static void
@@ -1584,7 +1602,7 @@ OpenTemporaryFile(bool interXact)
 	 * open it, if we'll be registering it below.
 	 */
 	if (!interXact)
-		ResourceOwnerEnlargeFiles(CurrentResourceOwner);
+		ResourceOwnerEnlarge(CurrentResourceOwner);
 
 	/*
 	 * If some temp tablespace(s) have been given to us, try to use the next
@@ -1714,7 +1732,7 @@ PathNameCreateTemporaryFile(const char *path, bool error_on_failure)
 {
 	File		file;
 
-	ResourceOwnerEnlargeFiles(CurrentResourceOwner);
+	ResourceOwnerEnlarge(CurrentResourceOwner);
 
 	/*
 	 * Open the file.  Note: we don't use O_EXCL, in case there is an orphaned
@@ -1752,7 +1770,7 @@ PathNameOpenTemporaryFile(const char *path, int mode)
 {
 	File		file;
 
-	ResourceOwnerEnlargeFiles(CurrentResourceOwner);
+	ResourceOwnerEnlarge(CurrentResourceOwner);
 
 	file = PathNameOpenFile(path, mode | PG_BINARY);
 
@@ -3604,3 +3622,19 @@ data_sync_elevel(int elevel)
 {
 	return data_sync_retry ? elevel : PANIC;
 }
+
+/*
+ * ResourceOwner callbacks
+ */
+static void
+ResOwnerReleaseFile(Datum res)
+{
+	FileClose((File) DatumGetInt32(res));
+}
+
+static void
+ResOwnerPrintFileLeakWarning(Datum res)
+{
+	elog(WARNING, "temporary file leak: File %d still referenced",
+		 DatumGetInt32(res));
+}
diff --git a/src/backend/storage/ipc/dsm.c b/src/backend/storage/ipc/dsm.c
index dffbd8e82a2..c665bc6cecf 100644
--- a/src/backend/storage/ipc/dsm.c
+++ b/src/backend/storage/ipc/dsm.c
@@ -37,13 +37,15 @@
 #include "miscadmin.h"
 #include "port/pg_bitutils.h"
 #include "storage/dsm.h"
+#include "storage/fd.h"
 #include "storage/ipc.h"
 #include "storage/lwlock.h"
 #include "storage/pg_shmem.h"
+#include "storage/shmem.h"
 #include "utils/freepage.h"
 #include "utils/guc.h"
 #include "utils/memutils.h"
-#include "utils/resowner_private.h"
+#include "utils/resowner.h"
 
 #define PG_DYNSHMEM_CONTROL_MAGIC		0x9a503d32
 
@@ -139,6 +141,25 @@ static dsm_control_header *dsm_control;
 static Size dsm_control_mapped_size = 0;
 static void *dsm_control_impl_private = NULL;
 
+
+/* ResourceOwner callbacks to hold DSM segments */
+static void ResOwnerReleaseDSM(Datum res);
+static void ResOwnerPrintDSMLeakWarning(Datum res);
+
+static ResourceOwnerFuncs dsm_resowner_funcs =
+{
+	.name = "dynamic shared memory segment",
+	.phase = RESOURCE_RELEASE_BEFORE_LOCKS,
+	.ReleaseResource = ResOwnerReleaseDSM,
+	.PrintLeakWarning = ResOwnerPrintDSMLeakWarning
+};
+
+/* Convenience wrappers over ResourceOwnerRemember/Forget */
+#define ResourceOwnerRememberDSM(owner, seg) \
+	ResourceOwnerRemember(owner, PointerGetDatum(seg), &dsm_resowner_funcs)
+#define ResourceOwnerForgetDSM(owner, seg) \
+	ResourceOwnerForget(owner, PointerGetDatum(seg), &dsm_resowner_funcs)
+
 /*
  * Start up the dynamic shared memory system.
  *
@@ -895,7 +916,7 @@ void
 dsm_unpin_mapping(dsm_segment *seg)
 {
 	Assert(seg->resowner == NULL);
-	ResourceOwnerEnlargeDSMs(CurrentResourceOwner);
+	ResourceOwnerEnlarge(CurrentResourceOwner);
 	seg->resowner = CurrentResourceOwner;
 	ResourceOwnerRememberDSM(seg->resowner, seg);
 }
@@ -1162,7 +1183,7 @@ dsm_create_descriptor(void)
 	dsm_segment *seg;
 
 	if (CurrentResourceOwner)
-		ResourceOwnerEnlargeDSMs(CurrentResourceOwner);
+		ResourceOwnerEnlarge(CurrentResourceOwner);
 
 	seg = MemoryContextAlloc(TopMemoryContext, sizeof(dsm_segment));
 	dlist_push_head(&dsm_segment_list, &seg->node);
@@ -1241,3 +1262,20 @@ is_main_region_dsm_handle(dsm_handle handle)
 {
 	return handle & 1;
 }
+
+/*
+ * ResourceOwner callbacks
+ */
+static void
+ResOwnerReleaseDSM(Datum res)
+{
+	dsm_detach((dsm_segment *) DatumGetPointer(res));
+}
+static void
+ResOwnerPrintDSMLeakWarning(Datum res)
+{
+	dsm_segment *seg = (dsm_segment *) res;
+
+	elog(WARNING, "dynamic shared memory leak: segment %u still referenced",
+		 dsm_segment_handle(seg));
+}
diff --git a/src/backend/storage/lmgr/lock.c b/src/backend/storage/lmgr/lock.c
index d86566f4554..98b4725c406 100644
--- a/src/backend/storage/lmgr/lock.c
+++ b/src/backend/storage/lmgr/lock.c
@@ -47,7 +47,7 @@
 #include "storage/standby.h"
 #include "utils/memutils.h"
 #include "utils/ps_status.h"
-#include "utils/resowner_private.h"
+#include "utils/resowner.h"
 
 
 /* This configuration variable is used to set the lock table size */
diff --git a/src/backend/utils/cache/catcache.c b/src/backend/utils/cache/catcache.c
index 3613ae5f44d..c9ebf9effe1 100644
--- a/src/backend/utils/cache/catcache.c
+++ b/src/backend/utils/cache/catcache.c
@@ -31,12 +31,13 @@
 #endif
 #include "storage/lmgr.h"
 #include "utils/builtins.h"
+#include "utils/catcache.h"
 #include "utils/datum.h"
 #include "utils/fmgroids.h"
 #include "utils/inval.h"
 #include "utils/memutils.h"
 #include "utils/rel.h"
-#include "utils/resowner_private.h"
+#include "utils/resowner.h"
 #include "utils/syscache.h"
 
 
@@ -104,6 +105,66 @@ static void CatCacheCopyKeys(TupleDesc tupdesc, int nkeys, int *attnos,
  *					internal support functions
  */
 
+/* ResourceOwner callbacks to hold catcache references */
+
+static void ResOwnerReleaseCatCache(Datum res);
+static void ResOwnerPrintCatCacheLeakWarning(Datum res);
+static void ResOwnerReleaseCatCacheList(Datum res);
+static void ResOwnerPrintCatCacheListLeakWarning(Datum res);
+
+static ResourceOwnerFuncs catcache_funcs =
+{
+	/* catcache references */
+	.name = "catcache reference",
+	.phase = RESOURCE_RELEASE_AFTER_LOCKS,
+	.ReleaseResource = ResOwnerReleaseCatCache,
+	.PrintLeakWarning = ResOwnerPrintCatCacheLeakWarning
+};
+
+static ResourceOwnerFuncs catlistref_funcs =
+{
+	/* catcache-list pins */
+	.name = "catcache list reference",
+	.phase = RESOURCE_RELEASE_AFTER_LOCKS,
+	.ReleaseResource = ResOwnerReleaseCatCacheList,
+	.PrintLeakWarning = ResOwnerPrintCatCacheListLeakWarning
+};
+
+/* support for catcache refcount management */
+static inline void
+ResourceOwnerEnlargeCatCacheRefs(ResourceOwner owner)
+{
+	ResourceOwnerEnlarge(owner);
+}
+static inline void
+ResourceOwnerRememberCatCacheRef(ResourceOwner owner, HeapTuple tuple)
+{
+	ResourceOwnerRemember(owner, PointerGetDatum(tuple), &catcache_funcs);
+}
+static inline void
+ResourceOwnerForgetCatCacheRef(ResourceOwner owner, HeapTuple tuple)
+{
+	ResourceOwnerForget(owner, PointerGetDatum(tuple), &catcache_funcs);
+}
+
+static inline void
+ResourceOwnerEnlargeCatCacheListRefs(ResourceOwner owner)
+{
+	ResourceOwnerEnlarge(owner);
+}
+static inline void
+ResourceOwnerRememberCatCacheListRef(ResourceOwner owner, CatCList *list)
+{
+	ResourceOwnerRemember(owner, PointerGetDatum(list), &catlistref_funcs);
+}
+
+static inline void
+ResourceOwnerForgetCatCacheListRef(ResourceOwner owner, CatCList *list)
+{
+	ResourceOwnerForget(owner, PointerGetDatum(list), &catlistref_funcs);
+}
+
+
 /*
  * Hash and equality functions for system types that are used as cache key
  * fields.  In some cases, we just call the regular SQL-callable functions for
@@ -1270,7 +1331,7 @@ SearchCatCacheInternal(CatCache *cache,
 		 */
 		if (!ct->negative)
 		{
-			ResourceOwnerEnlargeCatCacheRefs(CurrentResourceOwner);
+			ResourceOwnerEnlarge(CurrentResourceOwner);
 			ct->refcount++;
 			ResourceOwnerRememberCatCacheRef(CurrentResourceOwner, &ct->tuple);
 
@@ -1371,7 +1432,7 @@ SearchCatCacheMiss(CatCache *cache,
 									 hashValue, hashIndex,
 									 false);
 		/* immediately set the refcount to 1 */
-		ResourceOwnerEnlargeCatCacheRefs(CurrentResourceOwner);
+		ResourceOwnerEnlarge(CurrentResourceOwner);
 		ct->refcount++;
 		ResourceOwnerRememberCatCacheRef(CurrentResourceOwner, &ct->tuple);
 		break;					/* assume only one match */
@@ -1583,7 +1644,7 @@ SearchCatCacheList(CatCache *cache,
 		dlist_move_head(&cache->cc_lists, &cl->cache_elem);
 
 		/* Bump the list's refcount and return it */
-		ResourceOwnerEnlargeCatCacheListRefs(CurrentResourceOwner);
+		ResourceOwnerEnlarge(CurrentResourceOwner);
 		cl->refcount++;
 		ResourceOwnerRememberCatCacheListRef(CurrentResourceOwner, cl);
 
@@ -1607,7 +1668,7 @@ SearchCatCacheList(CatCache *cache,
 	 * block to ensure we can undo those refcounts if we get an error before
 	 * we finish constructing the CatCList.
 	 */
-	ResourceOwnerEnlargeCatCacheListRefs(CurrentResourceOwner);
+	ResourceOwnerEnlarge(CurrentResourceOwner);
 
 	ctlist = NIL;
 
@@ -2062,14 +2123,19 @@ PrepareToInvalidateCacheTuple(Relation relation,
 	}
 }
 
-
 /*
- * Subroutines for warning about reference leaks.  These are exported so
- * that resowner.c can call them.
+ * ResourceOwner callbacks
  */
-void
-PrintCatCacheLeakWarning(HeapTuple tuple)
+static void
+ResOwnerReleaseCatCache(Datum res)
 {
+	ReleaseCatCache((HeapTuple) DatumGetPointer(res));
+}
+
+static void
+ResOwnerPrintCatCacheLeakWarning(Datum res)
+{
+	HeapTuple tuple = (HeapTuple) DatumGetPointer(res);
 	CatCTup    *ct = (CatCTup *) (((char *) tuple) -
 								  offsetof(CatCTup, tuple));
 
@@ -2083,9 +2149,17 @@ PrintCatCacheLeakWarning(HeapTuple tuple)
 		 ct->refcount);
 }
 
-void
-PrintCatCacheListLeakWarning(CatCList *list)
+static void
+ResOwnerReleaseCatCacheList(Datum res)
+{
+	ReleaseCatCacheList((CatCList *) DatumGetPointer(res));
+}
+
+static void
+ResOwnerPrintCatCacheListLeakWarning(Datum res)
 {
+	CatCList *list = (CatCList *) DatumGetPointer(res);
+
 	elog(WARNING, "cache reference leak: cache %s (%d), list %p has count %d",
 		 list->my_cache->cc_relname, list->my_cache->id,
 		 list, list->refcount);
diff --git a/src/backend/utils/cache/plancache.c b/src/backend/utils/cache/plancache.c
index 50d6ad28b4c..8bce9d16ed5 100644
--- a/src/backend/utils/cache/plancache.c
+++ b/src/backend/utils/cache/plancache.c
@@ -69,7 +69,7 @@
 #include "tcop/utility.h"
 #include "utils/inval.h"
 #include "utils/memutils.h"
-#include "utils/resowner_private.h"
+#include "utils/resowner.h"
 #include "utils/rls.h"
 #include "utils/snapmgr.h"
 #include "utils/syscache.h"
@@ -115,6 +115,31 @@ static void PlanCacheRelCallback(Datum arg, Oid relid);
 static void PlanCacheObjectCallback(Datum arg, int cacheid, uint32 hashvalue);
 static void PlanCacheSysCallback(Datum arg, int cacheid, uint32 hashvalue);
 
+/* ResourceOwner callbacks to track plancache references */
+static void ResOwnerReleaseCachedPlan(Datum res);
+static void ResOwnerPrintPlanCacheLeakWarning(Datum res);
+
+/* this is exported for ResourceOwnerReleaseAllPlanCacheRefs() */
+ResourceOwnerFuncs planref_resowner_funcs =
+{
+	.name = "plancache reference",
+	.phase = RESOURCE_RELEASE_AFTER_LOCKS,
+	.ReleaseResource = ResOwnerReleaseCachedPlan,
+	.PrintLeakWarning = ResOwnerPrintPlanCacheLeakWarning
+};
+
+static inline void
+ResourceOwnerRememberPlanCacheRef(ResourceOwner owner, CachedPlan *plan)
+{
+	ResourceOwnerRemember(owner, PointerGetDatum(plan), &planref_resowner_funcs);
+}
+static inline void
+ResourceOwnerForgetPlanCacheRef(ResourceOwner owner, CachedPlan *plan)
+{
+	ResourceOwnerForget(owner, PointerGetDatum(plan), &planref_resowner_funcs);
+}
+
+
 /* GUC parameter */
 int			plan_cache_mode;
 
@@ -1229,7 +1254,7 @@ GetCachedPlan(CachedPlanSource *plansource, ParamListInfo boundParams,
 
 	/* Flag the plan as in use by caller */
 	if (useResOwner)
-		ResourceOwnerEnlargePlanCacheRefs(CurrentResourceOwner);
+		ResourceOwnerEnlarge(CurrentResourceOwner);
 	plan->refcount++;
 	if (useResOwner)
 		ResourceOwnerRememberPlanCacheRef(CurrentResourceOwner, plan);
@@ -1392,7 +1417,7 @@ CachedPlanAllowsSimpleValidityCheck(CachedPlanSource *plansource,
 	/* Bump refcount if requested. */
 	if (owner)
 	{
-		ResourceOwnerEnlargePlanCacheRefs(owner);
+		ResourceOwnerEnlarge(owner);
 		plan->refcount++;
 		ResourceOwnerRememberPlanCacheRef(owner, plan);
 	}
@@ -1451,7 +1476,7 @@ CachedPlanIsSimplyValid(CachedPlanSource *plansource, CachedPlan *plan,
 	/* It's still good.  Bump refcount if requested. */
 	if (owner)
 	{
-		ResourceOwnerEnlargePlanCacheRefs(owner);
+		ResourceOwnerEnlarge(owner);
 		plan->refcount++;
 		ResourceOwnerRememberPlanCacheRef(owner, plan);
 	}
@@ -2205,3 +2230,20 @@ ResetPlanCache(void)
 		cexpr->is_valid = false;
 	}
 }
+
+/*
+ * ResourceOwner callbacks
+ */
+
+static void
+ResOwnerReleaseCachedPlan(Datum res)
+{
+	ReleaseCachedPlan((CachedPlan *) DatumGetPointer(res), true);
+}
+
+static void
+ResOwnerPrintPlanCacheLeakWarning(Datum res)
+{
+	elog(WARNING, "plancache reference leak: plan %p not closed",
+		 DatumGetPointer(res));
+}
diff --git a/src/backend/utils/cache/relcache.c b/src/backend/utils/cache/relcache.c
index 66393becfb6..efff30e9238 100644
--- a/src/backend/utils/cache/relcache.c
+++ b/src/backend/utils/cache/relcache.c
@@ -78,13 +78,14 @@
 #include "storage/smgr.h"
 #include "utils/array.h"
 #include "utils/builtins.h"
+#include "utils/catcache.h"
 #include "utils/datum.h"
 #include "utils/fmgroids.h"
 #include "utils/inval.h"
 #include "utils/lsyscache.h"
 #include "utils/memutils.h"
 #include "utils/relmapper.h"
-#include "utils/resowner_private.h"
+#include "utils/resowner.h"
 #include "utils/snapmgr.h"
 #include "utils/syscache.h"
 
@@ -2066,6 +2067,18 @@ RelationIdGetRelation(Oid relationId)
  * ----------------------------------------------------------------
  */
 
+/* ResourceOwner callbacks to track relcache references */
+static void ResOwnerReleaseRelation(Datum res);
+static void ResOwnerPrintRelCacheLeakWarning(Datum res);
+
+static ResourceOwnerFuncs relref_resowner_funcs =
+{
+	.name = "relcache reference",
+	.phase = RESOURCE_RELEASE_BEFORE_LOCKS,
+	.ReleaseResource = ResOwnerReleaseRelation,
+	.PrintLeakWarning = ResOwnerPrintRelCacheLeakWarning
+};
+
 /*
  * RelationIncrementReferenceCount
  *		Increments relation reference count.
@@ -2077,10 +2090,10 @@ RelationIdGetRelation(Oid relationId)
 void
 RelationIncrementReferenceCount(Relation rel)
 {
-	ResourceOwnerEnlargeRelationRefs(CurrentResourceOwner);
+	ResourceOwnerEnlarge(CurrentResourceOwner);
 	rel->rd_refcnt += 1;
 	if (!IsBootstrapProcessingMode())
-		ResourceOwnerRememberRelationRef(CurrentResourceOwner, rel);
+		ResourceOwnerRemember(CurrentResourceOwner, PointerGetDatum(rel), &relref_resowner_funcs);
 }
 
 /*
@@ -2093,7 +2106,7 @@ RelationDecrementReferenceCount(Relation rel)
 	Assert(rel->rd_refcnt > 0);
 	rel->rd_refcnt -= 1;
 	if (!IsBootstrapProcessingMode())
-		ResourceOwnerForgetRelationRef(CurrentResourceOwner, rel);
+		ResourceOwnerForget(CurrentResourceOwner, PointerGetDatum(rel), &relref_resowner_funcs);
 }
 
 /*
@@ -6406,3 +6419,21 @@ unlink_initfile(const char *initfilename, int elevel)
 							initfilename)));
 	}
 }
+
+/*
+ * ResourceOwner callbacks
+ */
+static void
+ResOwnerPrintRelCacheLeakWarning(Datum res)
+{
+	Relation rel = (Relation) res;
+
+	elog(WARNING, "relcache reference leak: relation \"%s\" not closed",
+		 RelationGetRelationName(rel));
+}
+
+static void
+ResOwnerReleaseRelation(Datum res)
+{
+	RelationClose((Relation) res);
+}
diff --git a/src/backend/utils/cache/syscache.c b/src/backend/utils/cache/syscache.c
index 809b27a038f..a3c4063e655 100644
--- a/src/backend/utils/cache/syscache.c
+++ b/src/backend/utils/cache/syscache.c
@@ -982,7 +982,7 @@ static const struct cachedesc cacheinfo[] = {
 	}
 };
 
-static CatCache *SysCache[SysCacheSize];
+ CatCache *SysCache[SysCacheSize];
 
 static bool CacheInitialized = false;
 
diff --git a/src/backend/utils/resowner/README b/src/backend/utils/resowner/README
index 2998f6bb362..890db7d1e66 100644
--- a/src/backend/utils/resowner/README
+++ b/src/backend/utils/resowner/README
@@ -60,13 +60,18 @@ subtransaction or portal.  Therefore, the "release" operation on a child
 ResourceOwner transfers lock ownership to the parent instead of actually
 releasing the lock, if isCommit is true.
 
-Currently, ResourceOwners contain direct support for recording ownership of
-buffer pins, lmgr locks, and catcache, relcache, plancache, tupdesc, and
-snapshot references.  Other objects can be associated with a ResourceOwner by
-recording the address of the owning ResourceOwner in such an object.  There is
-an API for other modules to get control during ResourceOwner release, so that
-they can scan their own data structures to find the objects that need to be
-deleted.
+ResourceOwner can record ownership of many different kinds of objects.
+As of this writing, it's used internally for buffer pins, lmgr locks, and
+catcache, relcache, plancache, tupdesc, snapshot, DSM and JIT context references.
+ResourceOwner treates all objects the same, but to register a new kind of
+an object with it, you need to fill in a few callback functions, see
+ResourceOwnerFuncs.
+
+There is also an API for other modules to get control during ResourceOwner
+release, so that they can scan their own data structures to find the objects
+that need to be deleted. This used to be the only way to register new kinds
+of objects with a resource owner; nowadays it easier to write custom
+ResourceOwnerFuncs callabacks.
 
 Whenever we are inside a transaction, the global variable
 CurrentResourceOwner shows which resource owner should be assigned
diff --git a/src/backend/utils/resowner/resowner.c b/src/backend/utils/resowner/resowner.c
index 8bc2c4e9ea3..2e1ae4f8e0c 100644
--- a/src/backend/utils/resowner/resowner.c
+++ b/src/backend/utils/resowner/resowner.c
@@ -21,73 +21,44 @@
 #include "postgres.h"
 
 #include "common/hashfn.h"
-#include "jit/jit.h"
-#include "storage/bufmgr.h"
 #include "storage/ipc.h"
 #include "storage/predicate.h"
 #include "storage/proc.h"
 #include "utils/memutils.h"
-#include "utils/rel.h"
-#include "utils/resowner_private.h"
-#include "utils/snapmgr.h"
-
+#include "utils/plancache.h"
+#include "utils/resowner.h"
 
 /*
- * All resource IDs managed by this code are required to fit into a Datum,
- * which is fine since they are generally pointers or integers.
- *
- * Provide Datum conversion macros for a couple of things that are really
- * just "int".
- */
-#define FileGetDatum(file) Int32GetDatum(file)
-#define DatumGetFile(datum) ((File) DatumGetInt32(datum))
-#define BufferGetDatum(buffer) Int32GetDatum(buffer)
-#define DatumGetBuffer(datum) ((Buffer) DatumGetInt32(datum))
-
-/*
- * ResourceArray is a common structure for storing all types of resource IDs.
- *
- * We manage small sets of resource IDs by keeping them in a simple array:
- * itemsarr[k] holds an ID, for 0 <= k < nitems <= maxitems = capacity.
+ * ResourceElem represents a reference associated with a resource owner.
  *
- * If a set grows large, we switch over to using open-addressing hashing.
- * Then, itemsarr[] is a hash table of "capacity" slots, with each
- * slot holding either an ID or "invalidval".  nitems is the number of valid
- * items present; if it would exceed maxitems, we enlarge the array and
- * re-hash.  In this mode, maxitems should be rather less than capacity so
- * that we don't waste too much time searching for empty slots.
- *
- * In either mode, lastidx remembers the location of the last item inserted
- * or returned by GetAny; this speeds up searches in ResourceArrayRemove.
+ * All objects managed by this code are required to fit into a Datum,
+ * which is fine since they are generally pointers or integers.
  */
-typedef struct ResourceArray
+typedef struct ResourceElem
 {
-	Datum	   *itemsarr;		/* buffer for storing values */
-	Datum		invalidval;		/* value that is considered invalid */
-	uint32		capacity;		/* allocated length of itemsarr[] */
-	uint32		nitems;			/* how many items are stored in items array */
-	uint32		maxitems;		/* current limit on nitems before enlarging */
-	uint32		lastidx;		/* index of last item returned by GetAny */
-} ResourceArray;
+	Datum		item;
+	ResourceOwnerFuncs *kind;
+} ResourceElem;
 
 /*
- * Initially allocated size of a ResourceArray.  Must be power of two since
- * we'll use (arraysize - 1) as mask for hashing.
+ * Size of the small fixed-size array to hold most-recently remembered resources.
  */
-#define RESARRAY_INIT_SIZE 16
+#define RESOWNER_ARRAY_SIZE 8
 
 /*
- * When to switch to hashing vs. simple array logic in a ResourceArray.
+ * Initially allocated size of a ResourceOwner's hash.  Must be power of two since
+ * we'll use (capacity - 1) as mask for hashing.
  */
-#define RESARRAY_MAX_ARRAY 64
-#define RESARRAY_IS_ARRAY(resarr) ((resarr)->capacity <= RESARRAY_MAX_ARRAY)
+#define RESOWNER_HASH_INIT_SIZE 32
 
 /*
- * How many items may be stored in a resource array of given capacity.
+ * How many items may be stored in a hash of given capacity.
  * When this number is reached, we must resize.
  */
-#define RESARRAY_MAX_ITEMS(capacity) \
-	((capacity) <= RESARRAY_MAX_ARRAY ? (capacity) : (capacity)/4 * 3)
+#define RESOWNER_HASH_MAX_ITEMS(capacity) ((capacity)/4 * 3)
+
+StaticAssertDecl(RESOWNER_HASH_MAX_ITEMS(RESOWNER_HASH_INIT_SIZE) > RESOWNER_ARRAY_SIZE,
+				 "initial hash size too small compared to array size");
 
 /*
  * To speed up bulk releasing or reassigning locks from a resource owner to
@@ -117,22 +88,33 @@ typedef struct ResourceOwnerData
 	ResourceOwner nextchild;	/* next child of same parent */
 	const char *name;			/* name (just for debugging) */
 
-	/* We have built-in support for remembering: */
-	ResourceArray bufferarr;	/* owned buffers */
-	ResourceArray catrefarr;	/* catcache references */
-	ResourceArray catlistrefarr;	/* catcache-list pins */
-	ResourceArray relrefarr;	/* relcache references */
-	ResourceArray planrefarr;	/* plancache references */
-	ResourceArray tupdescarr;	/* tupdesc references */
-	ResourceArray snapshotarr;	/* snapshot references */
-	ResourceArray filearr;		/* open temporary files */
-	ResourceArray dsmarr;		/* dynamic shmem segments */
-	ResourceArray jitarr;		/* JIT contexts */
+	/*
+	 * These structs keep track of the objects registered with this owner.
+	 *
+	 * We manage a small set of references by keeping them in a simple
+	 * array. When the array gets full, all the elements in the array are
+	 * moved to a hash table. This way, the array always contains a few
+	 * most recently remembered references. To find a particular reference,
+	 * you need to search both the array and the hash table.
+	 */
+	ResourceElem arr[RESOWNER_ARRAY_SIZE];
+	uint32		narr;			/* how many items are stored in the array */
+
+	/*
+	 * The hash table. Uses open-addressing. 'nhash' is the number of items
+	 * present; if it would exceed 'grow_at', we enlarge it and re-hash.
+	 * 'grow_at' should be rather less than 'capacity' so that we don't waste
+	 * too much time searching for empty slots.
+	 */
+	ResourceElem *hash;
+	uint32		nhash;			/* how many items are stored in the hash */
+	uint32		capacity;		/* allocated length of hash[] */
+	uint32		grow_at;		/* grow hash when reach this */
 
 	/* We can remember up to MAX_RESOWNER_LOCKS references to local locks. */
 	int			nlocks;			/* number of owned locks */
 	LOCALLOCK  *locks[MAX_RESOWNER_LOCKS];	/* list of owned locks */
-}			ResourceOwnerData;
+} ResourceOwnerData;
 
 
 /*****************************************************************************
@@ -144,6 +126,18 @@ ResourceOwner CurTransactionResourceOwner = NULL;
 ResourceOwner TopTransactionResourceOwner = NULL;
 ResourceOwner AuxProcessResourceOwner = NULL;
 
+/* #define RESOWNER_STATS */
+/* #define RESOWNER_TRACE */
+
+#ifdef RESOWNER_STATS
+static int narray_lookups = 0;
+static int nhash_lookups = 0;
+#endif
+
+#ifdef RESOWNER_TRACE
+static int resowner_trace_counter = 0;
+#endif
+
 /*
  * List of add-on callbacks for resource releasing
  */
@@ -158,44 +152,36 @@ static ResourceReleaseCallbackItem *ResourceRelease_callbacks = NULL;
 
 
 /* Internal routines */
-static void ResourceArrayInit(ResourceArray *resarr, Datum invalidval);
-static void ResourceArrayEnlarge(ResourceArray *resarr);
-static void ResourceArrayAdd(ResourceArray *resarr, Datum value);
-static bool ResourceArrayRemove(ResourceArray *resarr, Datum value);
-static bool ResourceArrayGetAny(ResourceArray *resarr, Datum *value);
-static void ResourceArrayFree(ResourceArray *resarr);
 static void ResourceOwnerReleaseInternal(ResourceOwner owner,
 										 ResourceReleasePhase phase,
 										 bool isCommit,
 										 bool isTopLevel);
 static void ReleaseAuxProcessResourcesCallback(int code, Datum arg);
-static void PrintRelCacheLeakWarning(Relation rel);
-static void PrintPlanCacheLeakWarning(CachedPlan *plan);
-static void PrintTupleDescLeakWarning(TupleDesc tupdesc);
-static void PrintSnapshotLeakWarning(Snapshot snapshot);
-static void PrintFileLeakWarning(File file);
-static void PrintDSMLeakWarning(dsm_segment *seg);
+
+
 
 
 /*****************************************************************************
  *	  INTERNAL ROUTINES														 *
  *****************************************************************************/
 
-
-/*
- * Initialize a ResourceArray
- */
 static void
-ResourceArrayInit(ResourceArray *resarr, Datum invalidval)
+ResourceArrayAddToHash(ResourceOwner owner, Datum value, ResourceOwnerFuncs *kind)
 {
-	/* Assert it's empty */
-	Assert(resarr->itemsarr == NULL);
-	Assert(resarr->capacity == 0);
-	Assert(resarr->nitems == 0);
-	Assert(resarr->maxitems == 0);
-	/* Remember the appropriate "invalid" value */
-	resarr->invalidval = invalidval;
-	/* We don't allocate any storage until needed */
+	/* Insert into first free slot at or after hash location. */
+	uint32		mask = owner->capacity - 1;
+	uint32		idx;
+
+	idx = DatumGetUInt32(hash_any((void *) &value, sizeof(value))) & mask;
+	for (;;)
+	{
+		if (owner->hash[idx].kind == NULL)
+			break;
+		idx = (idx + 1) & mask;
+	}
+	owner->hash[idx].item = value;
+	owner->hash[idx].kind = kind;
+	owner->nhash++;
 }
 
 /*
@@ -204,205 +190,222 @@ ResourceArrayInit(ResourceArray *resarr, Datum invalidval)
  * This is separate from actually inserting a resource because if we run out
  * of memory, it's critical to do so *before* acquiring the resource.
  */
-static void
-ResourceArrayEnlarge(ResourceArray *resarr)
+void
+ResourceOwnerEnlarge(ResourceOwner owner)
 {
-	uint32		i,
-				oldcap,
-				newcap;
-	Datum	   *olditemsarr;
-	Datum	   *newitemsarr;
-
-	if (resarr->nitems < resarr->maxitems)
+	if (owner->narr < RESOWNER_ARRAY_SIZE)
 		return;					/* no work needed */
 
-	olditemsarr = resarr->itemsarr;
-	oldcap = resarr->capacity;
+	/* Is there space in the hash? If not, enlarge it. */
 
 	/* Double the capacity of the array (capacity must stay a power of 2!) */
-	newcap = (oldcap > 0) ? oldcap * 2 : RESARRAY_INIT_SIZE;
-	newitemsarr = (Datum *) MemoryContextAlloc(TopMemoryContext,
-											   newcap * sizeof(Datum));
-	for (i = 0; i < newcap; i++)
-		newitemsarr[i] = resarr->invalidval;
-
-	/* We assume we can't fail below this point, so OK to scribble on resarr */
-	resarr->itemsarr = newitemsarr;
-	resarr->capacity = newcap;
-	resarr->maxitems = RESARRAY_MAX_ITEMS(newcap);
-	resarr->nitems = 0;
-
-	if (olditemsarr != NULL)
+	if (owner->narr + owner->nhash >= owner->grow_at)
 	{
-		/*
-		 * Transfer any pre-existing entries into the new array; they don't
-		 * necessarily go where they were before, so this simple logic is the
-		 * best way.  Note that if we were managing the set as a simple array,
-		 * the entries after nitems are garbage, but that shouldn't matter
-		 * because we won't get here unless nitems was equal to oldcap.
-		 */
-		for (i = 0; i < oldcap; i++)
+		uint32		i,
+					oldcap,
+					newcap;
+		ResourceElem *oldhash;
+		ResourceElem *newhash;
+
+		oldhash = owner->hash;
+		oldcap = owner->capacity;
+
+		newcap = (oldcap > 0) ? oldcap * 2 : RESOWNER_HASH_INIT_SIZE;
+		newhash = (ResourceElem *) MemoryContextAllocZero(TopMemoryContext,
+														  newcap * sizeof(ResourceElem));
+
+		/* We assume we can't fail below this point, so OK to scribble on FIXME */
+		owner->hash = newhash;
+		owner->capacity = newcap;
+		owner->grow_at = RESOWNER_HASH_MAX_ITEMS(newcap);
+		owner->nhash = 0;
+
+		if (oldhash != NULL)
 		{
-			if (olditemsarr[i] != resarr->invalidval)
-				ResourceArrayAdd(resarr, olditemsarr[i]);
+			/*
+			 * Transfer any pre-existing entries into the new array; they don't
+			 * necessarily go where they were before, so this simple logic is the
+			 * best way.
+			 */
+			for (i = 0; i < oldcap; i++)
+			{
+				if (oldhash[i].kind != NULL)
+					ResourceArrayAddToHash(owner, oldhash[i].item, oldhash[i].kind);
+			}
+
+			/* And release old array. */
+			pfree(oldhash);
 		}
+	}
 
-		/* And release old array. */
-		pfree(olditemsarr);
+	/* Move items from the array to the hash */
+	for (int i = 0; i < owner->narr; i++)
+	{
+		ResourceArrayAddToHash(owner, owner->arr[i].item, owner->arr[i].kind);
 	}
+	owner->narr = 0;
 
-	Assert(resarr->nitems < resarr->maxitems);
+	Assert(owner->nhash < owner->grow_at);
 }
 
 /*
- * Add a resource to ResourceArray
+ * Remember that an object is owner by a ReourceOwner
  *
- * Caller must have previously done ResourceArrayEnlarge()
+ * Caller must have previously done ResourceOwnerEnlarge()
  */
-static void
-ResourceArrayAdd(ResourceArray *resarr, Datum value)
+void
+ResourceOwnerRemember(ResourceOwner owner, Datum value, ResourceOwnerFuncs *kind)
 {
 	uint32		idx;
 
-	Assert(value != resarr->invalidval);
-	Assert(resarr->nitems < resarr->maxitems);
+#ifdef RESOWNER_TRACE
+	elog(LOG, "REMEMBER %d: owner %p value " UINT64_FORMAT ", kind: %s",
+		 resowner_trace_counter++, owner, DatumGetUInt64(value), kind->name);
+#endif
 
-	if (RESARRAY_IS_ARRAY(resarr))
-	{
-		/* Append to linear array. */
-		idx = resarr->nitems;
-	}
-	else
-	{
-		/* Insert into first free slot at or after hash location. */
-		uint32		mask = resarr->capacity - 1;
+	Assert(owner->narr < RESOWNER_ARRAY_SIZE);
 
-		idx = DatumGetUInt32(hash_any((void *) &value, sizeof(value))) & mask;
-		for (;;)
-		{
-			if (resarr->itemsarr[idx] == resarr->invalidval)
-				break;
-			idx = (idx + 1) & mask;
-		}
-	}
-	resarr->lastidx = idx;
-	resarr->itemsarr[idx] = value;
-	resarr->nitems++;
+	/* Append to linear array. */
+	idx = owner->narr;
+	owner->arr[idx].item = value;
+	owner->arr[idx].kind = kind;
+	owner->narr++;
 }
 
 /*
- * Remove a resource from ResourceArray
+ * Forget that an object is owned by a ResourceOwner
  *
- * Returns true on success, false if resource was not found.
+ * Returns true on success. If the resource was not found, returns false,
+ * and calls kind->ForgetError callback.
  *
- * Note: if same resource ID appears more than once, one instance is removed.
+ * Note: if same resource ID is associated with the ResourceOwner more than once,
+ * one instance is removed.
  */
-static bool
-ResourceArrayRemove(ResourceArray *resarr, Datum value)
+void
+ResourceOwnerForget(ResourceOwner owner, Datum value, ResourceOwnerFuncs *kind)
 {
 	uint32		i,
-				idx,
-				lastidx = resarr->lastidx;
+				idx;
 
-	Assert(value != resarr->invalidval);
+#ifdef RESOWNER_TRACE
+	elog(LOG, "FORGET %d: owner %p value " UINT64_FORMAT ", kind: %s",
+		 resowner_trace_counter++, owner, DatumGetUInt64(value), kind->name);
+#endif
 
-	/* Search through all items, but try lastidx first. */
-	if (RESARRAY_IS_ARRAY(resarr))
+	/* Search through all items, but check the array first. */
+	for (i = 0; i < owner->narr; i++)
 	{
-		if (lastidx < resarr->nitems &&
-			resarr->itemsarr[lastidx] == value)
-		{
-			resarr->itemsarr[lastidx] = resarr->itemsarr[resarr->nitems - 1];
-			resarr->nitems--;
-			/* Update lastidx to make reverse-order removals fast. */
-			resarr->lastidx = resarr->nitems - 1;
-			return true;
-		}
-		for (i = 0; i < resarr->nitems; i++)
+		if (owner->arr[i].item == value &&
+			owner->arr[i].kind == kind)
 		{
-			if (resarr->itemsarr[i] == value)
-			{
-				resarr->itemsarr[i] = resarr->itemsarr[resarr->nitems - 1];
-				resarr->nitems--;
-				/* Update lastidx to make reverse-order removals fast. */
-				resarr->lastidx = resarr->nitems - 1;
-				return true;
-			}
+			owner->arr[i] = owner->arr[owner->narr - 1];
+			owner->narr--;
+
+#ifdef RESOWNER_STATS
+			narray_lookups++;
+#endif
+
+			return;
 		}
 	}
-	else
+
+	/* Search hash */
+	if (owner->nhash > 0)
 	{
-		uint32		mask = resarr->capacity - 1;
+		uint32		mask = owner->capacity - 1;
 
-		if (lastidx < resarr->capacity &&
-			resarr->itemsarr[lastidx] == value)
-		{
-			resarr->itemsarr[lastidx] = resarr->invalidval;
-			resarr->nitems--;
-			return true;
-		}
 		idx = DatumGetUInt32(hash_any((void *) &value, sizeof(value))) & mask;
-		for (i = 0; i < resarr->capacity; i++)
+		for (i = 0; i < owner->capacity; i++)
 		{
-			if (resarr->itemsarr[idx] == value)
+			if (owner->hash[idx].item == value &&
+				owner->hash[idx].kind == kind)
 			{
-				resarr->itemsarr[idx] = resarr->invalidval;
-				resarr->nitems--;
-				return true;
+				owner->hash[idx].item = (Datum) 0;
+				owner->hash[idx].kind = NULL;
+				owner->nhash--;
+
+#ifdef RESOWNER_STATS
+				nhash_lookups++;
+#endif
+				return;
 			}
 			idx = (idx + 1) & mask;
 		}
 	}
 
-	return false;
+	/*
+	 * Use %p to print the reference, since most objects tracked by a resource owner
+	 * are pointers. It's a bit misleading if it's not a pointer, but this is a
+	 * programmer error, anyway.
+	 */
+	elog(ERROR, "%s %p is not owned by resource owner %s",
+		 kind->name, DatumGetPointer(value), ResourceOwnerGetName(owner));
 }
 
 /*
- * Get any convenient entry in a ResourceArray.
- *
- * "Convenient" is defined as "easy for ResourceArrayRemove to remove";
- * we help that along by setting lastidx to match.  This avoids O(N^2) cost
- * when removing all ResourceArray items during ResourceOwner destruction.
- *
- * Returns true if we found an element, or false if the array is empty.
+ * Call the ReleaseResource callback on entries with given 'phase'.
  */
-static bool
-ResourceArrayGetAny(ResourceArray *resarr, Datum *value)
+static void
+ResourceOwnerReleaseAll(ResourceOwner owner, ResourceReleasePhase phase,
+						bool printLeakWarnings)
 {
-	if (resarr->nitems == 0)
-		return false;
+	bool		found;
 
-	if (RESARRAY_IS_ARRAY(resarr))
-	{
-		/* Linear array: just return the first element. */
-		resarr->lastidx = 0;
-	}
-	else
+	/* First handle all the entries in the array. */
+	do
 	{
-		/* Hash: search forward from wherever we were last. */
-		uint32		mask = resarr->capacity - 1;
+		found = false;
 
-		for (;;)
+		for (int i = 0; i < owner->narr; i++)
 		{
-			resarr->lastidx &= mask;
-			if (resarr->itemsarr[resarr->lastidx] != resarr->invalidval)
-				break;
-			resarr->lastidx++;
+			if (owner->arr[i].kind->phase == phase)
+			{
+				Datum		value = owner->arr[i].item;
+				ResourceOwnerFuncs *kind = owner->arr[i].kind;
+
+				if (printLeakWarnings)
+					kind->PrintLeakWarning(value);
+				kind->ReleaseResource(value);
+				found = true;
+			}
 		}
-	}
 
-	*value = resarr->itemsarr[resarr->lastidx];
-	return true;
-}
+		/*
+		 * If any resources were released, check again because some of the
+		 * elements might have moved by the callbacks. We don't want to
+		 * miss them.
+		 */
+	} while (found && owner->narr > 0);
 
-/*
- * Trash a ResourceArray (we don't care about its state after this)
- */
-static void
-ResourceArrayFree(ResourceArray *resarr)
-{
-	if (resarr->itemsarr)
-		pfree(resarr->itemsarr);
+	/* Ok, the array has now been handled. Then the hash */
+	do
+	{
+		found = false;
+
+		for (int idx = 0; idx < owner->capacity; idx++)
+		{
+			if (owner->hash[idx].kind != NULL &&
+				owner->hash[idx].kind->phase == phase)
+			{
+				/* found one */
+				Datum		value = owner->hash[idx].item;
+				ResourceOwnerFuncs *kind = owner->hash[idx].kind;
+
+				if (printLeakWarnings)
+					kind->PrintLeakWarning(value);
+				kind->ReleaseResource(value);
+				found = true;
+			}
+		}
+		/*
+		 * Like with the array, we must check again after we reach the
+		 * end, if any callbacks were called. XXX: We could probably
+		 * stipulate that the callbacks may not do certain thing, like
+		 * remember more references in the same resource owner, to avoid
+		 * that.
+		 */
+	}
+	while (found && owner->nhash > 0);
 }
 
 
@@ -434,16 +437,10 @@ ResourceOwnerCreate(ResourceOwner parent, const char *name)
 		parent->firstchild = owner;
 	}
 
-	ResourceArrayInit(&(owner->bufferarr), BufferGetDatum(InvalidBuffer));
-	ResourceArrayInit(&(owner->catrefarr), PointerGetDatum(NULL));
-	ResourceArrayInit(&(owner->catlistrefarr), PointerGetDatum(NULL));
-	ResourceArrayInit(&(owner->relrefarr), PointerGetDatum(NULL));
-	ResourceArrayInit(&(owner->planrefarr), PointerGetDatum(NULL));
-	ResourceArrayInit(&(owner->tupdescarr), PointerGetDatum(NULL));
-	ResourceArrayInit(&(owner->snapshotarr), PointerGetDatum(NULL));
-	ResourceArrayInit(&(owner->filearr), FileGetDatum(-1));
-	ResourceArrayInit(&(owner->dsmarr), PointerGetDatum(NULL));
-	ResourceArrayInit(&(owner->jitarr), PointerGetDatum(NULL));
+#ifdef RESOWNER_TRACE
+	elog(LOG, "CREATE %d: %p %s",
+		 resowner_trace_counter++, owner, name);
+#endif
 
 	return owner;
 }
@@ -482,6 +479,15 @@ ResourceOwnerRelease(ResourceOwner owner,
 {
 	/* There's not currently any setup needed before recursing */
 	ResourceOwnerReleaseInternal(owner, phase, isCommit, isTopLevel);
+
+#ifdef RESOWNER_STATS
+	if (isTopLevel)
+	{
+		elog(LOG, "RESOWNER STATS: lookups: array %d, hash %d", narray_lookups, nhash_lookups);
+		narray_lookups = 0;
+		nhash_lookups = 0;
+	}
+#endif
 }
 
 static void
@@ -493,7 +499,6 @@ ResourceOwnerReleaseInternal(ResourceOwner owner,
 	ResourceOwner child;
 	ResourceOwner save;
 	ResourceReleaseCallbackItem *item;
-	Datum		foundres;
 
 	/* Recurse to handle descendants */
 	for (child = owner->firstchild; child != NULL; child = child->nextchild)
@@ -509,50 +514,13 @@ ResourceOwnerReleaseInternal(ResourceOwner owner,
 	if (phase == RESOURCE_RELEASE_BEFORE_LOCKS)
 	{
 		/*
-		 * Release buffer pins.  Note that ReleaseBuffer will remove the
-		 * buffer entry from our array, so we just have to iterate till there
-		 * are none.
+		 * Release all references that need to be released before the locks.
 		 *
-		 * During a commit, there shouldn't be any remaining pins --- that
+		 * During a commit, there shouldn't be any remaining references --- that
 		 * would indicate failure to clean up the executor correctly --- so
 		 * issue warnings.  In the abort case, just clean up quietly.
 		 */
-		while (ResourceArrayGetAny(&(owner->bufferarr), &foundres))
-		{
-			Buffer		res = DatumGetBuffer(foundres);
-
-			if (isCommit)
-				PrintBufferLeakWarning(res);
-			ReleaseBuffer(res);
-		}
-
-		/* Ditto for relcache references */
-		while (ResourceArrayGetAny(&(owner->relrefarr), &foundres))
-		{
-			Relation	res = (Relation) DatumGetPointer(foundres);
-
-			if (isCommit)
-				PrintRelCacheLeakWarning(res);
-			RelationClose(res);
-		}
-
-		/* Ditto for dynamic shared memory segments */
-		while (ResourceArrayGetAny(&(owner->dsmarr), &foundres))
-		{
-			dsm_segment *res = (dsm_segment *) DatumGetPointer(foundres);
-
-			if (isCommit)
-				PrintDSMLeakWarning(res);
-			dsm_detach(res);
-		}
-
-		/* Ditto for JIT contexts */
-		while (ResourceArrayGetAny(&(owner->jitarr), &foundres))
-		{
-			JitContext *context = (JitContext *) PointerGetDatum(foundres);
-
-			jit_release_context(context);
-		}
+		ResourceOwnerReleaseAll(owner, phase, isCommit);
 	}
 	else if (phase == RESOURCE_RELEASE_LOCKS)
 	{
@@ -561,7 +529,7 @@ ResourceOwnerReleaseInternal(ResourceOwner owner,
 			/*
 			 * For a top-level xact we are going to release all locks (or at
 			 * least all non-session locks), so just do a single lmgr call at
-			 * the top of the recursion.
+			 * the top of the recursion
 			 */
 			if (owner == TopTransactionResourceOwner)
 			{
@@ -605,70 +573,9 @@ ResourceOwnerReleaseInternal(ResourceOwner owner,
 	else if (phase == RESOURCE_RELEASE_AFTER_LOCKS)
 	{
 		/*
-		 * Release catcache references.  Note that ReleaseCatCache will remove
-		 * the catref entry from our array, so we just have to iterate till
-		 * there are none.
-		 *
-		 * As with buffer pins, warn if any are left at commit time.
+		 * Release all references that need to be released after the locks.
 		 */
-		while (ResourceArrayGetAny(&(owner->catrefarr), &foundres))
-		{
-			HeapTuple	res = (HeapTuple) DatumGetPointer(foundres);
-
-			if (isCommit)
-				PrintCatCacheLeakWarning(res);
-			ReleaseCatCache(res);
-		}
-
-		/* Ditto for catcache lists */
-		while (ResourceArrayGetAny(&(owner->catlistrefarr), &foundres))
-		{
-			CatCList   *res = (CatCList *) DatumGetPointer(foundres);
-
-			if (isCommit)
-				PrintCatCacheListLeakWarning(res);
-			ReleaseCatCacheList(res);
-		}
-
-		/* Ditto for plancache references */
-		while (ResourceArrayGetAny(&(owner->planrefarr), &foundres))
-		{
-			CachedPlan *res = (CachedPlan *) DatumGetPointer(foundres);
-
-			if (isCommit)
-				PrintPlanCacheLeakWarning(res);
-			ReleaseCachedPlan(res, true);
-		}
-
-		/* Ditto for tupdesc references */
-		while (ResourceArrayGetAny(&(owner->tupdescarr), &foundres))
-		{
-			TupleDesc	res = (TupleDesc) DatumGetPointer(foundres);
-
-			if (isCommit)
-				PrintTupleDescLeakWarning(res);
-			DecrTupleDescRefCount(res);
-		}
-
-		/* Ditto for snapshot references */
-		while (ResourceArrayGetAny(&(owner->snapshotarr), &foundres))
-		{
-			Snapshot	res = (Snapshot) DatumGetPointer(foundres);
-
-			if (isCommit)
-				PrintSnapshotLeakWarning(res);
-			UnregisterSnapshot(res);
-		}
-
-		/* Ditto for temporary files */
-		while (ResourceArrayGetAny(&(owner->filearr), &foundres))
-		{
-			File		res = DatumGetFile(foundres);
-
-			if (isCommit)
-				PrintFileLeakWarning(res);
-			FileClose(res);
-		}
+		ResourceOwnerReleaseAll(owner, phase, isCommit);
 	}
 
 	/* Let add-on modules get a chance too */
@@ -689,16 +596,42 @@ void
 ResourceOwnerReleaseAllPlanCacheRefs(ResourceOwner owner)
 {
 	ResourceOwner save;
-	Datum		foundres;
 
 	save = CurrentResourceOwner;
 	CurrentResourceOwner = owner;
-	while (ResourceArrayGetAny(&(owner->planrefarr), &foundres))
+
+	/* array first */
+	for (int i = 0; i < owner->narr; i++)
+	{
+		if (owner->arr[i].kind == &planref_resowner_funcs)
+		{
+			CachedPlan *planref = (CachedPlan *) DatumGetPointer(owner->arr[i].item);
+
+			owner->arr[i] = owner->arr[owner->narr - 1];
+			owner->narr--;
+			i--;
+
+			/* pass 'false' because we already removed the entry from the resowner */
+			ReleaseCachedPlan(planref, false);
+		}
+	}
+
+	/* Then hash */
+	for (int i = 0; i < owner->capacity; i++)
 	{
-		CachedPlan *res = (CachedPlan *) DatumGetPointer(foundres);
+		if (owner->hash[i].kind == &planref_resowner_funcs)
+		{
+			CachedPlan *planref = (CachedPlan *) DatumGetPointer(owner->hash[i].item);
+
+			owner->hash[i].item = (Datum) 0;
+			owner->hash[i].kind = NULL;
+			owner->nhash--;
 
-		ReleaseCachedPlan(res, true);
+			/* pass 'false' because we already removed the entry from the resowner */
+			ReleaseCachedPlan(planref, false);
+		}
 	}
+
 	CurrentResourceOwner = save;
 }
 
@@ -715,18 +648,15 @@ ResourceOwnerDelete(ResourceOwner owner)
 	Assert(owner != CurrentResourceOwner);
 
 	/* And it better not own any resources, either */
-	Assert(owner->bufferarr.nitems == 0);
-	Assert(owner->catrefarr.nitems == 0);
-	Assert(owner->catlistrefarr.nitems == 0);
-	Assert(owner->relrefarr.nitems == 0);
-	Assert(owner->planrefarr.nitems == 0);
-	Assert(owner->tupdescarr.nitems == 0);
-	Assert(owner->snapshotarr.nitems == 0);
-	Assert(owner->filearr.nitems == 0);
-	Assert(owner->dsmarr.nitems == 0);
-	Assert(owner->jitarr.nitems == 0);
+	Assert(owner->narr == 0);
+	Assert(owner->nhash == 0);
 	Assert(owner->nlocks == 0 || owner->nlocks == MAX_RESOWNER_LOCKS + 1);
 
+#ifdef RESOWNER_TRACE
+	elog(LOG, "DELETE %d: %p %s",
+		 resowner_trace_counter++, owner, owner->name);
+#endif
+
 	/*
 	 * Delete children.  The recursive call will delink the child from me, so
 	 * just iterate as long as there is a child.
@@ -742,17 +672,8 @@ ResourceOwnerDelete(ResourceOwner owner)
 	ResourceOwnerNewParent(owner, NULL);
 
 	/* And free the object. */
-	ResourceArrayFree(&(owner->bufferarr));
-	ResourceArrayFree(&(owner->catrefarr));
-	ResourceArrayFree(&(owner->catlistrefarr));
-	ResourceArrayFree(&(owner->relrefarr));
-	ResourceArrayFree(&(owner->planrefarr));
-	ResourceArrayFree(&(owner->tupdescarr));
-	ResourceArrayFree(&(owner->snapshotarr));
-	ResourceArrayFree(&(owner->filearr));
-	ResourceArrayFree(&(owner->dsmarr));
-	ResourceArrayFree(&(owner->jitarr));
-
+	if (owner->hash)
+		pfree(owner->hash);
 	pfree(owner);
 }
 
@@ -807,6 +728,15 @@ ResourceOwnerNewParent(ResourceOwner owner,
 	}
 }
 
+/*
+ * Get the name of a resource owner, for debugging purposes
+ */
+const char *
+ResourceOwnerGetName(ResourceOwner owner)
+{
+	return owner->name;
+}
+
 /*
  * Register or deregister callback functions for resource cleanup
  *
@@ -905,44 +835,6 @@ ReleaseAuxProcessResourcesCallback(int code, Datum arg)
 	ReleaseAuxProcessResources(isCommit);
 }
 
-
-/*
- * Make sure there is room for at least one more entry in a ResourceOwner's
- * buffer array.
- *
- * This is separate from actually inserting an entry because if we run out
- * of memory, it's critical to do so *before* acquiring the resource.
- */
-void
-ResourceOwnerEnlargeBuffers(ResourceOwner owner)
-{
-	/* We used to allow pinning buffers without a resowner, but no more */
-	Assert(owner != NULL);
-	ResourceArrayEnlarge(&(owner->bufferarr));
-}
-
-/*
- * Remember that a buffer pin is owned by a ResourceOwner
- *
- * Caller must have previously done ResourceOwnerEnlargeBuffers()
- */
-void
-ResourceOwnerRememberBuffer(ResourceOwner owner, Buffer buffer)
-{
-	ResourceArrayAdd(&(owner->bufferarr), BufferGetDatum(buffer));
-}
-
-/*
- * Forget that a buffer pin is owned by a ResourceOwner
- */
-void
-ResourceOwnerForgetBuffer(ResourceOwner owner, Buffer buffer)
-{
-	if (!ResourceArrayRemove(&(owner->bufferarr), BufferGetDatum(buffer)))
-		elog(ERROR, "buffer %d is not owned by resource owner %s",
-			 buffer, owner->name);
-}
-
 /*
  * Remember that a Local Lock is owned by a ResourceOwner
  *
@@ -994,379 +886,3 @@ ResourceOwnerForgetLock(ResourceOwner owner, LOCALLOCK *locallock)
 	elog(ERROR, "lock reference %p is not owned by resource owner %s",
 		 locallock, owner->name);
 }
-
-/*
- * Make sure there is room for at least one more entry in a ResourceOwner's
- * catcache reference array.
- *
- * This is separate from actually inserting an entry because if we run out
- * of memory, it's critical to do so *before* acquiring the resource.
- */
-void
-ResourceOwnerEnlargeCatCacheRefs(ResourceOwner owner)
-{
-	ResourceArrayEnlarge(&(owner->catrefarr));
-}
-
-/*
- * Remember that a catcache reference is owned by a ResourceOwner
- *
- * Caller must have previously done ResourceOwnerEnlargeCatCacheRefs()
- */
-void
-ResourceOwnerRememberCatCacheRef(ResourceOwner owner, HeapTuple tuple)
-{
-	ResourceArrayAdd(&(owner->catrefarr), PointerGetDatum(tuple));
-}
-
-/*
- * Forget that a catcache reference is owned by a ResourceOwner
- */
-void
-ResourceOwnerForgetCatCacheRef(ResourceOwner owner, HeapTuple tuple)
-{
-	if (!ResourceArrayRemove(&(owner->catrefarr), PointerGetDatum(tuple)))
-		elog(ERROR, "catcache reference %p is not owned by resource owner %s",
-			 tuple, owner->name);
-}
-
-/*
- * Make sure there is room for at least one more entry in a ResourceOwner's
- * catcache-list reference array.
- *
- * This is separate from actually inserting an entry because if we run out
- * of memory, it's critical to do so *before* acquiring the resource.
- */
-void
-ResourceOwnerEnlargeCatCacheListRefs(ResourceOwner owner)
-{
-	ResourceArrayEnlarge(&(owner->catlistrefarr));
-}
-
-/*
- * Remember that a catcache-list reference is owned by a ResourceOwner
- *
- * Caller must have previously done ResourceOwnerEnlargeCatCacheListRefs()
- */
-void
-ResourceOwnerRememberCatCacheListRef(ResourceOwner owner, CatCList *list)
-{
-	ResourceArrayAdd(&(owner->catlistrefarr), PointerGetDatum(list));
-}
-
-/*
- * Forget that a catcache-list reference is owned by a ResourceOwner
- */
-void
-ResourceOwnerForgetCatCacheListRef(ResourceOwner owner, CatCList *list)
-{
-	if (!ResourceArrayRemove(&(owner->catlistrefarr), PointerGetDatum(list)))
-		elog(ERROR, "catcache list reference %p is not owned by resource owner %s",
-			 list, owner->name);
-}
-
-/*
- * Make sure there is room for at least one more entry in a ResourceOwner's
- * relcache reference array.
- *
- * This is separate from actually inserting an entry because if we run out
- * of memory, it's critical to do so *before* acquiring the resource.
- */
-void
-ResourceOwnerEnlargeRelationRefs(ResourceOwner owner)
-{
-	ResourceArrayEnlarge(&(owner->relrefarr));
-}
-
-/*
- * Remember that a relcache reference is owned by a ResourceOwner
- *
- * Caller must have previously done ResourceOwnerEnlargeRelationRefs()
- */
-void
-ResourceOwnerRememberRelationRef(ResourceOwner owner, Relation rel)
-{
-	ResourceArrayAdd(&(owner->relrefarr), PointerGetDatum(rel));
-}
-
-/*
- * Forget that a relcache reference is owned by a ResourceOwner
- */
-void
-ResourceOwnerForgetRelationRef(ResourceOwner owner, Relation rel)
-{
-	if (!ResourceArrayRemove(&(owner->relrefarr), PointerGetDatum(rel)))
-		elog(ERROR, "relcache reference %s is not owned by resource owner %s",
-			 RelationGetRelationName(rel), owner->name);
-}
-
-/*
- * Debugging subroutine
- */
-static void
-PrintRelCacheLeakWarning(Relation rel)
-{
-	elog(WARNING, "relcache reference leak: relation \"%s\" not closed",
-		 RelationGetRelationName(rel));
-}
-
-/*
- * Make sure there is room for at least one more entry in a ResourceOwner's
- * plancache reference array.
- *
- * This is separate from actually inserting an entry because if we run out
- * of memory, it's critical to do so *before* acquiring the resource.
- */
-void
-ResourceOwnerEnlargePlanCacheRefs(ResourceOwner owner)
-{
-	ResourceArrayEnlarge(&(owner->planrefarr));
-}
-
-/*
- * Remember that a plancache reference is owned by a ResourceOwner
- *
- * Caller must have previously done ResourceOwnerEnlargePlanCacheRefs()
- */
-void
-ResourceOwnerRememberPlanCacheRef(ResourceOwner owner, CachedPlan *plan)
-{
-	ResourceArrayAdd(&(owner->planrefarr), PointerGetDatum(plan));
-}
-
-/*
- * Forget that a plancache reference is owned by a ResourceOwner
- */
-void
-ResourceOwnerForgetPlanCacheRef(ResourceOwner owner, CachedPlan *plan)
-{
-	if (!ResourceArrayRemove(&(owner->planrefarr), PointerGetDatum(plan)))
-		elog(ERROR, "plancache reference %p is not owned by resource owner %s",
-			 plan, owner->name);
-}
-
-/*
- * Debugging subroutine
- */
-static void
-PrintPlanCacheLeakWarning(CachedPlan *plan)
-{
-	elog(WARNING, "plancache reference leak: plan %p not closed", plan);
-}
-
-/*
- * Make sure there is room for at least one more entry in a ResourceOwner's
- * tupdesc reference array.
- *
- * This is separate from actually inserting an entry because if we run out
- * of memory, it's critical to do so *before* acquiring the resource.
- */
-void
-ResourceOwnerEnlargeTupleDescs(ResourceOwner owner)
-{
-	ResourceArrayEnlarge(&(owner->tupdescarr));
-}
-
-/*
- * Remember that a tupdesc reference is owned by a ResourceOwner
- *
- * Caller must have previously done ResourceOwnerEnlargeTupleDescs()
- */
-void
-ResourceOwnerRememberTupleDesc(ResourceOwner owner, TupleDesc tupdesc)
-{
-	ResourceArrayAdd(&(owner->tupdescarr), PointerGetDatum(tupdesc));
-}
-
-/*
- * Forget that a tupdesc reference is owned by a ResourceOwner
- */
-void
-ResourceOwnerForgetTupleDesc(ResourceOwner owner, TupleDesc tupdesc)
-{
-	if (!ResourceArrayRemove(&(owner->tupdescarr), PointerGetDatum(tupdesc)))
-		elog(ERROR, "tupdesc reference %p is not owned by resource owner %s",
-			 tupdesc, owner->name);
-}
-
-/*
- * Debugging subroutine
- */
-static void
-PrintTupleDescLeakWarning(TupleDesc tupdesc)
-{
-	elog(WARNING,
-		 "TupleDesc reference leak: TupleDesc %p (%u,%d) still referenced",
-		 tupdesc, tupdesc->tdtypeid, tupdesc->tdtypmod);
-}
-
-/*
- * Make sure there is room for at least one more entry in a ResourceOwner's
- * snapshot reference array.
- *
- * This is separate from actually inserting an entry because if we run out
- * of memory, it's critical to do so *before* acquiring the resource.
- */
-void
-ResourceOwnerEnlargeSnapshots(ResourceOwner owner)
-{
-	ResourceArrayEnlarge(&(owner->snapshotarr));
-}
-
-/*
- * Remember that a snapshot reference is owned by a ResourceOwner
- *
- * Caller must have previously done ResourceOwnerEnlargeSnapshots()
- */
-void
-ResourceOwnerRememberSnapshot(ResourceOwner owner, Snapshot snapshot)
-{
-	ResourceArrayAdd(&(owner->snapshotarr), PointerGetDatum(snapshot));
-}
-
-/*
- * Forget that a snapshot reference is owned by a ResourceOwner
- */
-void
-ResourceOwnerForgetSnapshot(ResourceOwner owner, Snapshot snapshot)
-{
-	if (!ResourceArrayRemove(&(owner->snapshotarr), PointerGetDatum(snapshot)))
-		elog(ERROR, "snapshot reference %p is not owned by resource owner %s",
-			 snapshot, owner->name);
-}
-
-/*
- * Debugging subroutine
- */
-static void
-PrintSnapshotLeakWarning(Snapshot snapshot)
-{
-	elog(WARNING, "Snapshot reference leak: Snapshot %p still referenced",
-		 snapshot);
-}
-
-
-/*
- * Make sure there is room for at least one more entry in a ResourceOwner's
- * files reference array.
- *
- * This is separate from actually inserting an entry because if we run out
- * of memory, it's critical to do so *before* acquiring the resource.
- */
-void
-ResourceOwnerEnlargeFiles(ResourceOwner owner)
-{
-	ResourceArrayEnlarge(&(owner->filearr));
-}
-
-/*
- * Remember that a temporary file is owned by a ResourceOwner
- *
- * Caller must have previously done ResourceOwnerEnlargeFiles()
- */
-void
-ResourceOwnerRememberFile(ResourceOwner owner, File file)
-{
-	ResourceArrayAdd(&(owner->filearr), FileGetDatum(file));
-}
-
-/*
- * Forget that a temporary file is owned by a ResourceOwner
- */
-void
-ResourceOwnerForgetFile(ResourceOwner owner, File file)
-{
-	if (!ResourceArrayRemove(&(owner->filearr), FileGetDatum(file)))
-		elog(ERROR, "temporary file %d is not owned by resource owner %s",
-			 file, owner->name);
-}
-
-/*
- * Debugging subroutine
- */
-static void
-PrintFileLeakWarning(File file)
-{
-	elog(WARNING, "temporary file leak: File %d still referenced",
-		 file);
-}
-
-/*
- * Make sure there is room for at least one more entry in a ResourceOwner's
- * dynamic shmem segment reference array.
- *
- * This is separate from actually inserting an entry because if we run out
- * of memory, it's critical to do so *before* acquiring the resource.
- */
-void
-ResourceOwnerEnlargeDSMs(ResourceOwner owner)
-{
-	ResourceArrayEnlarge(&(owner->dsmarr));
-}
-
-/*
- * Remember that a dynamic shmem segment is owned by a ResourceOwner
- *
- * Caller must have previously done ResourceOwnerEnlargeDSMs()
- */
-void
-ResourceOwnerRememberDSM(ResourceOwner owner, dsm_segment *seg)
-{
-	ResourceArrayAdd(&(owner->dsmarr), PointerGetDatum(seg));
-}
-
-/*
- * Forget that a dynamic shmem segment is owned by a ResourceOwner
- */
-void
-ResourceOwnerForgetDSM(ResourceOwner owner, dsm_segment *seg)
-{
-	if (!ResourceArrayRemove(&(owner->dsmarr), PointerGetDatum(seg)))
-		elog(ERROR, "dynamic shared memory segment %u is not owned by resource owner %s",
-			 dsm_segment_handle(seg), owner->name);
-}
-
-/*
- * Debugging subroutine
- */
-static void
-PrintDSMLeakWarning(dsm_segment *seg)
-{
-	elog(WARNING, "dynamic shared memory leak: segment %u still referenced",
-		 dsm_segment_handle(seg));
-}
-
-/*
- * Make sure there is room for at least one more entry in a ResourceOwner's
- * JIT context reference array.
- *
- * This is separate from actually inserting an entry because if we run out of
- * memory, it's critical to do so *before* acquiring the resource.
- */
-void
-ResourceOwnerEnlargeJIT(ResourceOwner owner)
-{
-	ResourceArrayEnlarge(&(owner->jitarr));
-}
-
-/*
- * Remember that a JIT context is owned by a ResourceOwner
- *
- * Caller must have previously done ResourceOwnerEnlargeJIT()
- */
-void
-ResourceOwnerRememberJIT(ResourceOwner owner, Datum handle)
-{
-	ResourceArrayAdd(&(owner->jitarr), handle);
-}
-
-/*
- * Forget that a JIT context is owned by a ResourceOwner
- */
-void
-ResourceOwnerForgetJIT(ResourceOwner owner, Datum handle)
-{
-	if (!ResourceArrayRemove(&(owner->jitarr), handle))
-		elog(ERROR, "JIT context %p is not owned by resource owner %s",
-			 DatumGetPointer(handle), owner->name);
-}
diff --git a/src/backend/utils/time/snapmgr.c b/src/backend/utils/time/snapmgr.c
index 8c41483e87c..b3ea3ce7650 100644
--- a/src/backend/utils/time/snapmgr.c
+++ b/src/backend/utils/time/snapmgr.c
@@ -66,7 +66,7 @@
 #include "utils/memutils.h"
 #include "utils/old_snapshot.h"
 #include "utils/rel.h"
-#include "utils/resowner_private.h"
+#include "utils/resowner.h"
 #include "utils/snapmgr.h"
 #include "utils/syscache.h"
 #include "utils/timestamp.h"
@@ -174,6 +174,18 @@ static Snapshot CopySnapshot(Snapshot snapshot);
 static void FreeSnapshot(Snapshot snapshot);
 static void SnapshotResetXmin(void);
 
+/* ResourceOwner callbacks to track snapshot references */
+static void ResOwnerReleaseSnapshot(Datum res);
+static void ResOwnerPrintSnapshotLeakWarning(Datum res);
+
+static ResourceOwnerFuncs snapshot_resowner_funcs =
+{
+	.name = "snapshot reference",
+	.phase = RESOURCE_RELEASE_AFTER_LOCKS,
+	.ReleaseResource = ResOwnerReleaseSnapshot,
+	.PrintLeakWarning = ResOwnerPrintSnapshotLeakWarning
+};
+
 /*
  * Snapshot fields to be serialized.
  *
@@ -831,9 +843,10 @@ RegisterSnapshotOnOwner(Snapshot snapshot, ResourceOwner owner)
 	snap = snapshot->copied ? snapshot : CopySnapshot(snapshot);
 
 	/* and tell resowner.c about it */
-	ResourceOwnerEnlargeSnapshots(owner);
+	ResourceOwnerEnlarge(owner);
 	snap->regd_count++;
-	ResourceOwnerRememberSnapshot(owner, snap);
+	ResourceOwnerRemember(owner, PointerGetDatum(snap),
+						  &snapshot_resowner_funcs);
 
 	if (snap->regd_count == 1)
 		pairingheap_add(&RegisteredSnapshots, &snap->ph_node);
@@ -870,7 +883,8 @@ UnregisterSnapshotFromOwner(Snapshot snapshot, ResourceOwner owner)
 	Assert(snapshot->regd_count > 0);
 	Assert(!pairingheap_is_empty(&RegisteredSnapshots));
 
-	ResourceOwnerForgetSnapshot(owner, snapshot);
+	ResourceOwnerForget(owner, PointerGetDatum(snapshot),
+						&snapshot_resowner_funcs);
 
 	snapshot->regd_count--;
 	if (snapshot->regd_count == 0)
@@ -2345,3 +2359,19 @@ XidInMVCCSnapshot(TransactionId xid, Snapshot snapshot)
 
 	return false;
 }
+
+/*
+ * ResourceOwner callbacks
+ */
+static void
+ResOwnerReleaseSnapshot(Datum res)
+{
+	UnregisterSnapshot((Snapshot) DatumGetPointer(res));
+}
+
+static void
+ResOwnerPrintSnapshotLeakWarning(Datum res)
+{
+	elog(WARNING, "Snapshot reference leak: Snapshot %p still referenced",
+		 DatumGetPointer(res));
+}
diff --git a/src/include/storage/buf_internals.h b/src/include/storage/buf_internals.h
index 3377fa56768..c4aa7d7e31e 100644
--- a/src/include/storage/buf_internals.h
+++ b/src/include/storage/buf_internals.h
@@ -296,6 +296,21 @@ typedef struct CkptSortItem
 
 extern CkptSortItem *CkptBufferIds;
 
+/* ResourceOwner callbacks to hold buffer pins */
+extern ResourceOwnerFuncs buffer_resowner_funcs;
+
+/* Convenience wrappers over ResourceOwnerRemember/Forget */
+static inline void
+ResourceOwnerRememberBuffer(ResourceOwner owner, Buffer buffer)
+{
+	ResourceOwnerRemember(owner, Int32GetDatum(buffer), &buffer_resowner_funcs);
+}
+static inline void
+ResourceOwnerForgetBuffer(ResourceOwner owner, Buffer buffer)
+{
+	ResourceOwnerForget(owner, Int32GetDatum(buffer), &buffer_resowner_funcs);
+}
+
 /*
  * Internal buffer management routines
  */
diff --git a/src/include/utils/catcache.h b/src/include/utils/catcache.h
index f4aa316604e..dbb10dfdd16 100644
--- a/src/include/utils/catcache.h
+++ b/src/include/utils/catcache.h
@@ -225,7 +225,4 @@ extern void PrepareToInvalidateCacheTuple(Relation relation,
 										  HeapTuple newtuple,
 										  void (*function) (int, uint32, Oid));
 
-extern void PrintCatCacheLeakWarning(HeapTuple tuple);
-extern void PrintCatCacheListLeakWarning(CatCList *list);
-
 #endif							/* CATCACHE_H */
diff --git a/src/include/utils/plancache.h b/src/include/utils/plancache.h
index 4901568553c..2e0d5006719 100644
--- a/src/include/utils/plancache.h
+++ b/src/include/utils/plancache.h
@@ -233,4 +233,6 @@ extern bool CachedPlanIsSimplyValid(CachedPlanSource *plansource,
 extern CachedExpression *GetCachedExpression(Node *expr);
 extern void FreeCachedExpression(CachedExpression *cexpr);
 
+extern ResourceOwnerFuncs planref_resowner_funcs;
+
 #endif							/* PLANCACHE_H */
diff --git a/src/include/utils/resowner.h b/src/include/utils/resowner.h
index 878f39ccf14..87a2bf0558e 100644
--- a/src/include/utils/resowner.h
+++ b/src/include/utils/resowner.h
@@ -50,6 +50,32 @@ typedef enum
 	RESOURCE_RELEASE_AFTER_LOCKS
 } ResourceReleasePhase;
 
+/*
+ * In order to track an object, resowner.c needs a few callbacks for it.
+ * The callbacks for an object of a specific kind are encapsulated in
+ * ResourceOwnerFuncs.
+ */
+typedef struct ResourceOwnerFuncs
+{
+	const char *name;		/* name for the object kind, for debugging */
+	ResourceReleasePhase phase; /* when are these objects released? */
+
+	/*
+	 * Release resource.
+	 *
+	 * NOTE: this must call ResourceOwnerForget to disassociate it with the
+	 * resource owner.
+	 */
+	void (*ReleaseResource)(Datum res);
+
+	/*
+	 * Print a warning, when a resource has not been properly released before
+	 * commit.
+	 */
+	void (*PrintLeakWarning)(Datum res);
+
+} ResourceOwnerFuncs;
+
 /*
  *	Dynamically loaded modules can get control during ResourceOwnerRelease
  *	by providing a callback of this form.
@@ -71,16 +97,30 @@ extern void ResourceOwnerRelease(ResourceOwner owner,
 								 ResourceReleasePhase phase,
 								 bool isCommit,
 								 bool isTopLevel);
-extern void ResourceOwnerReleaseAllPlanCacheRefs(ResourceOwner owner);
 extern void ResourceOwnerDelete(ResourceOwner owner);
 extern ResourceOwner ResourceOwnerGetParent(ResourceOwner owner);
 extern void ResourceOwnerNewParent(ResourceOwner owner,
 								   ResourceOwner newparent);
+extern const char *ResourceOwnerGetName(ResourceOwner owner);
+
+extern void ResourceOwnerEnlarge(ResourceOwner owner);
+extern void ResourceOwnerRemember(ResourceOwner owner, Datum res, ResourceOwnerFuncs *kind);
+extern void ResourceOwnerForget(ResourceOwner owner, Datum res, ResourceOwnerFuncs *kind);
+
 extern void RegisterResourceReleaseCallback(ResourceReleaseCallback callback,
 											void *arg);
 extern void UnregisterResourceReleaseCallback(ResourceReleaseCallback callback,
 											  void *arg);
+
 extern void CreateAuxProcessResourceOwner(void);
 extern void ReleaseAuxProcessResources(bool isCommit);
 
+/* special support for local lock management */
+struct LOCALLOCK;
+extern void ResourceOwnerRememberLock(ResourceOwner owner, struct LOCALLOCK *locallock);
+extern void ResourceOwnerForgetLock(ResourceOwner owner, struct LOCALLOCK *locallock);
+
+/* special function to relase all plancache references */
+extern void ResourceOwnerReleaseAllPlanCacheRefs(ResourceOwner owner);
+
 #endif							/* RESOWNER_H */
diff --git a/src/include/utils/resowner_private.h b/src/include/utils/resowner_private.h
deleted file mode 100644
index a781a7a2aa6..00000000000
--- a/src/include/utils/resowner_private.h
+++ /dev/null
@@ -1,98 +0,0 @@
-/*-------------------------------------------------------------------------
- *
- * resowner_private.h
- *	  POSTGRES resource owner private definitions.
- *
- * See utils/resowner/README for more info.
- *
- *
- * Portions Copyright (c) 1996-2020, PostgreSQL Global Development Group
- * Portions Copyright (c) 1994, Regents of the University of California
- *
- * src/include/utils/resowner_private.h
- *
- *-------------------------------------------------------------------------
- */
-#ifndef RESOWNER_PRIVATE_H
-#define RESOWNER_PRIVATE_H
-
-#include "storage/dsm.h"
-#include "storage/fd.h"
-#include "storage/lock.h"
-#include "utils/catcache.h"
-#include "utils/plancache.h"
-#include "utils/resowner.h"
-#include "utils/snapshot.h"
-
-
-/* support for buffer refcount management */
-extern void ResourceOwnerEnlargeBuffers(ResourceOwner owner);
-extern void ResourceOwnerRememberBuffer(ResourceOwner owner, Buffer buffer);
-extern void ResourceOwnerForgetBuffer(ResourceOwner owner, Buffer buffer);
-
-/* support for local lock management */
-extern void ResourceOwnerRememberLock(ResourceOwner owner, LOCALLOCK *locallock);
-extern void ResourceOwnerForgetLock(ResourceOwner owner, LOCALLOCK *locallock);
-
-/* support for catcache refcount management */
-extern void ResourceOwnerEnlargeCatCacheRefs(ResourceOwner owner);
-extern void ResourceOwnerRememberCatCacheRef(ResourceOwner owner,
-											 HeapTuple tuple);
-extern void ResourceOwnerForgetCatCacheRef(ResourceOwner owner,
-										   HeapTuple tuple);
-extern void ResourceOwnerEnlargeCatCacheListRefs(ResourceOwner owner);
-extern void ResourceOwnerRememberCatCacheListRef(ResourceOwner owner,
-												 CatCList *list);
-extern void ResourceOwnerForgetCatCacheListRef(ResourceOwner owner,
-											   CatCList *list);
-
-/* support for relcache refcount management */
-extern void ResourceOwnerEnlargeRelationRefs(ResourceOwner owner);
-extern void ResourceOwnerRememberRelationRef(ResourceOwner owner,
-											 Relation rel);
-extern void ResourceOwnerForgetRelationRef(ResourceOwner owner,
-										   Relation rel);
-
-/* support for plancache refcount management */
-extern void ResourceOwnerEnlargePlanCacheRefs(ResourceOwner owner);
-extern void ResourceOwnerRememberPlanCacheRef(ResourceOwner owner,
-											  CachedPlan *plan);
-extern void ResourceOwnerForgetPlanCacheRef(ResourceOwner owner,
-											CachedPlan *plan);
-
-/* support for tupledesc refcount management */
-extern void ResourceOwnerEnlargeTupleDescs(ResourceOwner owner);
-extern void ResourceOwnerRememberTupleDesc(ResourceOwner owner,
-										   TupleDesc tupdesc);
-extern void ResourceOwnerForgetTupleDesc(ResourceOwner owner,
-										 TupleDesc tupdesc);
-
-/* support for snapshot refcount management */
-extern void ResourceOwnerEnlargeSnapshots(ResourceOwner owner);
-extern void ResourceOwnerRememberSnapshot(ResourceOwner owner,
-										  Snapshot snapshot);
-extern void ResourceOwnerForgetSnapshot(ResourceOwner owner,
-										Snapshot snapshot);
-
-/* support for temporary file management */
-extern void ResourceOwnerEnlargeFiles(ResourceOwner owner);
-extern void ResourceOwnerRememberFile(ResourceOwner owner,
-									  File file);
-extern void ResourceOwnerForgetFile(ResourceOwner owner,
-									File file);
-
-/* support for dynamic shared memory management */
-extern void ResourceOwnerEnlargeDSMs(ResourceOwner owner);
-extern void ResourceOwnerRememberDSM(ResourceOwner owner,
-									 dsm_segment *);
-extern void ResourceOwnerForgetDSM(ResourceOwner owner,
-								   dsm_segment *);
-
-/* support for JITContext management */
-extern void ResourceOwnerEnlargeJIT(ResourceOwner owner);
-extern void ResourceOwnerRememberJIT(ResourceOwner owner,
-									 Datum handle);
-extern void ResourceOwnerForgetJIT(ResourceOwner owner,
-								   Datum handle);
-
-#endif							/* RESOWNER_PRIVATE_H */


^ permalink  raw  reply  [nested|flat] 125+ messages in thread