Subject: [PATCH 4/5] Catcache pruning feature.

From: Kyotaro Horiguchi <[email-masked]>
Date: Mon, 1 Jul 2019 11:31:54 +0900

Currently we don't have a mechanism to limit the memory amount for
syscache. Syscache bloat often causes process die by OOM killer or
other problems. This patch lets old syscache entries removed to
eventually limit the amount of cache.

This patch intentionally unchanges indentation of an existing code
block in SearchCatCacheInternal for the patch size to be smaller. It
is adjusted in the next patch.
---
 src/backend/utils/cache/catcache.c | 186 +++++++++++++++++++++++++++++++++++++
 src/backend/utils/misc/guc.c       |  12 +++
 src/include/utils/catcache.h       |  17 ++++
 3 files changed, 215 insertions(+)

diff --git a/src/backend/utils/cache/catcache.c b/src/backend/utils/cache/catcache.c
index 98427b67cd..b552ae960c 100644
--- a/src/backend/utils/cache/catcache.c
+++ b/src/backend/utils/cache/catcache.c
@@ -60,9 +60,18 @@
 #define CACHE_elog(...)
 #endif
 
+/*
+ * GUC variable to define the minimum age of entries that will be considered
+ * to be evicted in seconds. -1 to disable the feature.
+ */
+int catalog_cache_prune_min_age = -1;
+
 /* Cache management header --- pointer is NULL until created */
 static CatCacheHeader *CacheHdr = NULL;
 
+/* Clock for the last accessed time of a catcache entry. */
+TimestampTz	catcacheclock = 0;
+
 static HeapTuple SearchCatCacheInternal(CatCache *cache,
 										int nkeys,
 										Datum v1, Datum v2,
@@ -864,9 +873,107 @@ InitCatCache(int id,
 	 */
 	MemoryContextSwitchTo(oldcxt);
 
+	/* initialize catcache reference clock if haven't done yet */
+	if (catcacheclock == 0)
+		catcacheclock = GetCurrentTimestamp();
+
+	/*
+	 * This cache doesn't contain a tuple older than the current time. Prevent
+	 * the first pruning from happening too early.
+	 */
+	cp->cc_oldest_ts = catcacheclock;
+
 	return cp;
 }
 
+/*
+ * CatCacheCleanupOldEntries - Remove infrequently-used entries
+ *
+ * Catcache entries happen to be left unused for a long time for several
+ * reasons. Remove such entries to prevent catcache from bloating. It is based
+ * on the similar algorithm with buffer eviction. Entries that are accessed
+ * several times in a certain period live longer than those that have had less
+ * access in the same duration.
+ */
+static bool
+CatCacheCleanupOldEntries(CatCache *cp)
+{
+	int		nremoved = 0;
+	int		i;
+	long	oldest_ts = catcacheclock;
+	long	age;
+	int		us;
+
+	/* Return immediately if disabled */
+	if (catalog_cache_prune_min_age < 0)
+		return false;
+
+	/* Don't scan the hash when we know we don't have prunable entries */
+	TimestampDifference(cp->cc_oldest_ts, catcacheclock, &age, &us);
+	if (age < catalog_cache_prune_min_age)
+		return false;
+
+	/* Scan over the whole hash to find entries to remove */
+	for (i = 0 ; i < cp->cc_nbuckets ; i++)
+	{
+		dlist_mutable_iter	iter;
+
+		dlist_foreach_modify(iter, &cp->cc_bucket[i])
+		{
+			CatCTup    *ct = dlist_container(CatCTup, cache_elem, iter.cur);
+
+			/* Don't remove referenced entries */
+			if (ct->refcount == 0 &&
+				(ct->c_list == NULL || ct->c_list->refcount == 0))
+			{
+				/*
+				 * Calculate the duration from the time from the last access
+				 * to the "current" time. catcacheclock is updated
+				 * per-statement basis and additionaly udpated periodically
+				 * during a long running query.
+				 */
+				TimestampDifference(ct->lastaccess, catcacheclock, &age, &us);
+
+				if (age > catalog_cache_prune_min_age)
+				{
+					/*
+					 * Entries that are not accessed after the last pruning
+					 * are removed in that seconds, and their lives are
+					 * prolonged according to how many times they are accessed
+					 * up to three times of the duration. We don't try shrink
+					 * buckets since pruning effectively caps catcache
+					 * expansion in the long term.
+					 */
+					if (ct->naccess > 2)
+						ct->naccess = 1;
+					else if (ct->naccess > 0)
+						ct->naccess--;
+					else
+					{
+						CatCacheRemoveCTup(cp, ct);
+						nremoved++;
+
+						/* don't update oldest_ts by removed entry */
+						continue;
+					}
+				}
+			}
+
+			/* update oldest timestamp if the entry remains alive */
+			if (ct->lastaccess < oldest_ts)
+				oldest_ts = ct->lastaccess;
+		}
+	}
+
+	cp->cc_oldest_ts = oldest_ts;
+
+	if (nremoved > 0)
+		elog(DEBUG1, "pruning catalog cache id=%d for %s: removed %d / %d",
+			 cp->id, cp->cc_relname, nremoved, cp->cc_ntup + nremoved);
+
+	return nremoved > 0;
+}
+
 /*
  * Enlarge a catcache, doubling the number of buckets.
  */
@@ -880,6 +987,10 @@ RehashCatCache(CatCache *cp)
 	elog(DEBUG1, "rehashing catalog cache id %d for %s; %d tups, %d buckets",
 		 cp->id, cp->cc_relname, cp->cc_ntup, cp->cc_nbuckets);
 
+	/* try removing old entries before expanding the hash */
+	if (CatCacheCleanupOldEntries(cp))
+		return;
+
 	/* Allocate a new, larger, hash table. */
 	newnbuckets = cp->cc_nbuckets * 2;
 	newbucket = (dlist_head *) MemoryContextAllocZero(CacheMemoryContext, newnbuckets * sizeof(dlist_head));
@@ -1257,6 +1368,14 @@ SearchCatCacheInternal(CatCache *cache,
 	 * dlist within the loop, because we don't continue the loop afterwards.
 	 */
 	bucket = &cache->cc_bucket[hashIndex];
+
+	/*
+	 * Even though this branch leads to duplicate of a bit much code, we want
+	 * as less branches as possible here to keep fastest when pruning is
+	 * disabled. Don't try to move this branch to foreach to save lines.
+	 */
+	if (likely(catalog_cache_prune_min_age < 0))
+	{
 	dlist_foreach(iter, bucket)
 	{
 		ct = dlist_container(CatCTup, cache_elem, iter.cur);
@@ -1309,6 +1428,71 @@ SearchCatCacheInternal(CatCache *cache,
 			return NULL;
 		}
 	}
+	}
+	else
+	{
+		/*
+		 * We manage the age of each entries for pruning in this branch.
+		 */
+		dlist_foreach(iter, bucket)
+		{
+			/* The following section is the same with the if() block */
+			ct = dlist_container(CatCTup, cache_elem, iter.cur);
+
+			if (ct->dead)
+				continue;
+
+			if (ct->hash_value != hashValue)
+				continue;
+
+			if (!CatalogCacheCompareTuple(cache, nkeys, ct->keys, arguments))
+				continue;
+
+			dlist_move_head(bucket, &ct->cache_elem);
+
+			/*
+			 * Prolong life of this entry. Since we want run as less
+			 * instructions as possible and want the branch be stable for
+			 * performance reasons, we don't give a strict cap on the
+			 * counter. All numbers above 1 will be regarded as 2 in
+			 * CatCacheCleanupOldEntries().
+			 */
+			ct->naccess++;
+			if (unlikely(ct->naccess == 0))
+				ct->naccess = 2;
+			ct->lastaccess = catcacheclock;
+
+			/* Following part is also the same with if() block above */
+			if (!ct->negative)
+			{
+				ResourceOwnerEnlargeCatCacheRefs(CurrentResourceOwner);
+				ct->refcount++;
+				ResourceOwnerRememberCatCacheRef(CurrentResourceOwner,
+												 &ct->tuple);
+
+				CACHE_elog(DEBUG2, "SearchCatCache(%s): found in bucket %d",
+						   cache->cc_relname, hashIndex);
+				
+#ifdef CATCACHE_STATS
+				cache->cc_hits++;
+#endif
+				
+
+				return &ct->tuple;
+			}
+			else
+			{
+				CACHE_elog(DEBUG2, "SearchCatCache(%s): found neg entry in bucket %d",
+						   cache->cc_relname, hashIndex);
+
+#ifdef CATCACHE_STATS
+				cache->cc_neg_hits++;
+#endif
+
+				return NULL;
+			}
+		}
+	}
 
 	return SearchCatCacheMiss(cache, nkeys, hashValue, hashIndex, v1, v2, v3, v4);
 }
@@ -1902,6 +2086,8 @@ CatalogCacheCreateEntry(CatCache *cache, HeapTuple ntp, Datum *arguments,
 	ct->dead = false;
 	ct->negative = negative;
 	ct->hash_value = hashValue;
+	ct->naccess = 0;
+	ct->lastaccess = catcacheclock;
 
 	dlist_push_head(&cache->cc_bucket[hashIndex], &ct->cache_elem);
 
diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c
index 92c4fee8f8..c2a4caa44b 100644
--- a/src/backend/utils/misc/guc.c
+++ b/src/backend/utils/misc/guc.c
@@ -82,6 +82,7 @@
 #include "tsearch/ts_cache.h"
 #include "utils/builtins.h"
 #include "utils/bytea.h"
+#include "utils/catcache.h"
 #include "utils/guc_tables.h"
 #include "utils/float.h"
 #include "utils/memutils.h"
@@ -2252,6 +2253,17 @@ static struct config_int ConfigureNamesInt[] =
 		NULL, NULL, NULL
 	},
 
+	{
+		{"catalog_cache_prune_min_age", PGC_USERSET, RESOURCES_MEM,
+			gettext_noop("System catalog cache entries that live unused for longer than this seconds are considered for removal."),
+			gettext_noop("The value of -1 turns off pruning."),
+			GUC_UNIT_S
+		},
+		&catalog_cache_prune_min_age,
+		300, -1, INT_MAX,
+		NULL, NULL, NULL
+	},
+
 	/*
 	 * We use the hopefully-safely-small value of 100kB as the compiled-in
 	 * default for max_stack_depth.  InitializeGUCOptions will increase it if
diff --git a/src/include/utils/catcache.h b/src/include/utils/catcache.h
index ff1fabaca1..ad962fb096 100644
--- a/src/include/utils/catcache.h
+++ b/src/include/utils/catcache.h
@@ -22,6 +22,7 @@
 
 #include "access/htup.h"
 #include "access/skey.h"
+#include "datatype/timestamp.h"
 #include "lib/ilist.h"
 #include "utils/relcache.h"
 
@@ -61,6 +62,7 @@ typedef struct catcache
 	slist_node	cc_next;		/* list link */
 	ScanKeyData cc_skey[CATCACHE_MAXKEYS];	/* precomputed key info for heap
 											 * scans */
+	TimestampTz	cc_oldest_ts;	/* timestamp of the oldest tuple in the hash */
 
 	/*
 	 * Keep these at the end, so that compiling catcache.c with CATCACHE_STATS
@@ -119,6 +121,8 @@ typedef struct catctup
 	bool		dead;			/* dead but not yet removed? */
 	bool		negative;		/* negative cache entry? */
 	HeapTupleData tuple;		/* tuple management header */
+	unsigned int naccess;		/* # of access to this entry */
+	TimestampTz	lastaccess;		/* timestamp of the last usage */
 
 	/*
 	 * The tuple may also be a member of at most one CatCList.  (If a single
@@ -189,6 +193,19 @@ typedef struct catcacheheader
 /* this extern duplicates utils/memutils.h... */
 extern PGDLLIMPORT MemoryContext CacheMemoryContext;
 
+/* for guc.c, not PGDLLPMPORT'ed */
+extern int catalog_cache_prune_min_age;
+
+/* source clock for access timestamp of catcache entries */
+extern TimestampTz catcacheclock;
+
+/* SetCatCacheClock - set catcache timestamp source clodk */
+static inline void
+SetCatCacheClock(TimestampTz ts)
+{
+	catcacheclock = ts;
+}
+
 extern void CreateCacheMemoryContext(void);
 
 extern CatCache *InitCatCache(int id, Oid reloid, Oid indexoid,
-- 
2.16.3

----Next_Part(Mon_Jul_01_16_02_59_2019_106)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v18-0005-Adjust-indentation-of-SearchCatCacheInternal.patch"