public inbox for [email protected]
help / color / mirror / Atom feedFrom: Kyotaro Horiguchi <[email protected]>
Subject: [PATCH] Remove entries that haven't been used for a certain time
Date: Fri, 1 Mar 2019 13:32:51 +0900
Catcache entries happen to be left alone for several reasons. It is
not desirable that such useless entries eat up memory. Catcache
pruning feature removes entries that haven't been accessed for a
certain time before enlarging hash array.
---
doc/src/sgml/config.sgml | 19 ++++
src/backend/tcop/postgres.c | 2 +
src/backend/utils/cache/catcache.c | 121 +++++++++++++++++++++++++-
src/backend/utils/misc/guc.c | 12 +++
src/backend/utils/misc/postgresql.conf.sample | 1 +
src/include/utils/catcache.h | 17 ++++
6 files changed, 169 insertions(+), 3 deletions(-)
diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index d383de2512..4231235447 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -1677,6 +1677,25 @@ include_dir 'conf.d'
</listitem>
</varlistentry>
+ <varlistentry id="guc-catalog-cache-prune-min-age" xreflabel="catalog_cache_prune_min_age">
+ <term><varname>catalog_cache_prune_min_age</varname> (<type>integer</type>)
+ <indexterm>
+ <primary><varname>catalog_cache_prune_min_age</varname> configuration
+ parameter</primary>
+ </indexterm>
+ </term>
+ <listitem>
+ <para>
+ Specifies the minimum amount of unused time in seconds at which a
+ system catalog cache entry is removed. -1 indicates that this feature
+ is disabled at all. The value defaults to 300 seconds (<literal>5
+ minutes</literal>). The entries that are not used for the duration
+ can be removed to prevent catalog cache from bloating with useless
+ entries.
+ </para>
+ </listitem>
+ </varlistentry>
+
<varlistentry id="guc-max-stack-depth" xreflabel="max_stack_depth">
<term><varname>max_stack_depth</varname> (<type>integer</type>)
<indexterm>
diff --git a/src/backend/tcop/postgres.c b/src/backend/tcop/postgres.c
index f9ce3d8f22..acab473d34 100644
--- a/src/backend/tcop/postgres.c
+++ b/src/backend/tcop/postgres.c
@@ -71,6 +71,7 @@
#include "tcop/pquery.h"
#include "tcop/tcopprot.h"
#include "tcop/utility.h"
+#include "utils/catcache.h"
#include "utils/lsyscache.h"
#include "utils/memutils.h"
#include "utils/ps_status.h"
@@ -2575,6 +2576,7 @@ start_xact_command(void)
* not desired, the timeout has to be disabled explicitly.
*/
enable_statement_timeout();
+ SetCatCacheClock(GetCurrentStatementStartTimestamp());
}
static void
diff --git a/src/backend/utils/cache/catcache.c b/src/backend/utils/cache/catcache.c
index d05930bc4c..c4582fe5a3 100644
--- a/src/backend/utils/cache/catcache.c
+++ b/src/backend/utils/cache/catcache.c
@@ -38,6 +38,7 @@
#include "utils/rel.h"
#include "utils/resowner_private.h"
#include "utils/syscache.h"
+#include "utils/timeout.h"
/* #define CACHEDEBUG */ /* turns DEBUG elogs on */
@@ -60,9 +61,18 @@
#define CACHE_elog(...)
#endif
+/*
+ * GUC variable to define the minimum age of entries that will be considered
+ * to be evicted in seconds. -1 to disable the feature.
+ */
+int catalog_cache_prune_min_age = 300;
+
/* Cache management header --- pointer is NULL until created */
static CatCacheHeader *CacheHdr = NULL;
+/* Clock for the last accessed time of a catcache entry. */
+TimestampTz catcacheclock = 0;
+
static inline HeapTuple SearchCatCacheInternal(CatCache *cache,
int nkeys,
Datum v1, Datum v2,
@@ -850,9 +860,99 @@ InitCatCache(int id,
*/
MemoryContextSwitchTo(oldcxt);
+ /* initialize catcache reference clock if haven't done yet */
+ if (catcacheclock == 0)
+ catcacheclock = GetCurrentTimestamp();
+
return cp;
}
+/*
+ * CatCacheCleanupOldEntries - Remove infrequently-used entries
+ *
+ * Catcache entries happen to be left unused for a long time for several
+ * reasons. Remove such entries to prevent catcache from bloating. It is based
+ * on the similar algorithm with buffer eviction. Entries that are accessed
+ * several times in a certain period live longer than those that have had less
+ * access in the same duration.
+ */
+static bool
+CatCacheCleanupOldEntries(CatCache *cp)
+{
+ int nremoved = 0;
+ int i;
+ long oldest_ts = catcacheclock;
+ long age;
+ int us;
+
+ /* Return immediately if disabled */
+ if (catalog_cache_prune_min_age == 0)
+ return false;
+
+ /* Don't scan the hash when we know we don't have prunable entries */
+ TimestampDifference(cp->cc_oldest_ts, catcacheclock, &age, &us);
+ if (age < catalog_cache_prune_min_age)
+ return false;
+
+ /* Scan over the whole hash to find entries to remove */
+ for (i = 0 ; i < cp->cc_nbuckets ; i++)
+ {
+ dlist_mutable_iter iter;
+
+ dlist_foreach_modify(iter, &cp->cc_bucket[i])
+ {
+ CatCTup *ct = dlist_container(CatCTup, cache_elem, iter.cur);
+
+ /* Don't remove referenced entries */
+ if (ct->refcount == 0 &&
+ (ct->c_list == NULL || ct->c_list->refcount == 0))
+ {
+ /*
+ * Calculate the duration from the time from the last access
+ * to the "current" time. catcacheclock is updated
+ * per-statement basis and additionaly udpated periodically
+ * during a long running query.
+ */
+ TimestampDifference(ct->lastaccess, catcacheclock, &age, &us);
+
+ if (age >= catalog_cache_prune_min_age)
+ {
+ /*
+ * Entries that are not accessed after the last pruning
+ * are removed in that seconds, and their lives are
+ * prolonged according to how many times they are accessed
+ * up to three times of the duration. We don't try shrink
+ * buckets since pruning effectively caps catcache
+ * expansion in the long term.
+ */
+ if (ct->naccess > 0)
+ ct->naccess--;
+ else
+ {
+ CatCacheRemoveCTup(cp, ct);
+ nremoved++;
+
+ /* don't update oldest_ts by removed entry */
+ continue;
+ }
+ }
+ }
+
+ /* update oldest timestamp if the entry remains alive */
+ if (ct->lastaccess < oldest_ts)
+ oldest_ts = ct->lastaccess;
+ }
+ }
+
+ cp->cc_oldest_ts = oldest_ts;
+
+ if (nremoved > 0)
+ elog(DEBUG1, "pruning catalog cache id=%d for %s: removed %d / %d",
+ cp->id, cp->cc_relname, nremoved, cp->cc_ntup + nremoved);
+
+ return nremoved > 0;
+}
+
/*
* Enlarge a catcache, doubling the number of buckets.
*/
@@ -1264,6 +1364,12 @@ SearchCatCacheInternal(CatCache *cache,
*/
dlist_move_head(bucket, &ct->cache_elem);
+ /* prolong life of this entry */
+ if (ct->naccess < 2)
+ ct->naccess++;
+
+ ct->lastaccess = catcacheclock;
+
/*
* If it's a positive entry, bump its refcount and return it. If it's
* negative, we can report failure to the caller.
@@ -1888,19 +1994,28 @@ CatalogCacheCreateEntry(CatCache *cache, HeapTuple ntp, Datum *arguments,
ct->dead = false;
ct->negative = negative;
ct->hash_value = hashValue;
+ ct->naccess = 0;
+ ct->lastaccess = catcacheclock;
dlist_push_head(&cache->cc_bucket[hashIndex], &ct->cache_elem);
cache->cc_ntup++;
CacheHdr->ch_ntup++;
+ /* increase refcount so that the new entry survives pruning */
+ ct->refcount++;
+
/*
- * If the hash table has become too full, enlarge the buckets array. Quite
- * arbitrarily, we enlarge when fill factor > 2.
+ * If the hash table has become too full, try removing infrequently used
+ * entries to make a room for the new entry. If failed, enlarge the bucket
+ * array instead. Quite arbitrarily, we try this when fill factor > 2.
*/
- if (cache->cc_ntup > cache->cc_nbuckets * 2)
+ if (cache->cc_ntup > cache->cc_nbuckets * 2 &&
+ !CatCacheCleanupOldEntries(cache))
RehashCatCache(cache);
+ ct->refcount--;
+
return ct;
}
diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c
index aa564d153a..e624c74bf9 100644
--- a/src/backend/utils/misc/guc.c
+++ b/src/backend/utils/misc/guc.c
@@ -82,6 +82,7 @@
#include "tsearch/ts_cache.h"
#include "utils/builtins.h"
#include "utils/bytea.h"
+#include "utils/catcache.h"
#include "utils/guc_tables.h"
#include "utils/float.h"
#include "utils/memutils.h"
@@ -2202,6 +2203,17 @@ static struct config_int ConfigureNamesInt[] =
NULL, NULL, NULL
},
+ {
+ {"catalog_cache_prune_min_age", PGC_USERSET, RESOURCES_MEM,
+ gettext_noop("System catalog cache entries that live unused for longer than this seconds are considered for removal."),
+ gettext_noop("The value of -1 turns off pruning."),
+ GUC_UNIT_S
+ },
+ &catalog_cache_prune_min_age,
+ 300, -1, INT_MAX,
+ NULL, NULL, NULL
+ },
+
/*
* We use the hopefully-safely-small value of 100kB as the compiled-in
* default for max_stack_depth. InitializeGUCOptions will increase it if
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index cccb5f145a..fa117f0573 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -128,6 +128,7 @@
#work_mem = 4MB # min 64kB
#maintenance_work_mem = 64MB # min 1MB
#autovacuum_work_mem = -1 # min 1MB, or -1 to use maintenance_work_mem
+#catalog_cache_prune_min_age = 300s # -1 disables pruning
#max_stack_depth = 2MB # min 100kB
#shared_memory_type = mmap # the default is the first option
# supported by the operating system:
diff --git a/src/include/utils/catcache.h b/src/include/utils/catcache.h
index 65d816a583..2f697d5ca4 100644
--- a/src/include/utils/catcache.h
+++ b/src/include/utils/catcache.h
@@ -22,6 +22,7 @@
#include "access/htup.h"
#include "access/skey.h"
+#include "datatype/timestamp.h"
#include "lib/ilist.h"
#include "utils/relcache.h"
@@ -61,6 +62,7 @@ typedef struct catcache
slist_node cc_next; /* list link */
ScanKeyData cc_skey[CATCACHE_MAXKEYS]; /* precomputed key info for heap
* scans */
+ long cc_oldest_ts;
/*
* Keep these at the end, so that compiling catcache.c with CATCACHE_STATS
@@ -119,6 +121,8 @@ typedef struct catctup
bool dead; /* dead but not yet removed? */
bool negative; /* negative cache entry? */
HeapTupleData tuple; /* tuple management header */
+ int naccess; /* # of access to this entry, up to 2 */
+ TimestampTz lastaccess; /* timestamp of the last usage */
/*
* The tuple may also be a member of at most one CatCList. (If a single
@@ -189,6 +193,19 @@ typedef struct catcacheheader
/* this extern duplicates utils/memutils.h... */
extern PGDLLIMPORT MemoryContext CacheMemoryContext;
+/* for guc.c, not PGDLLPMPORT'ed */
+extern int catalog_cache_prune_min_age;
+
+/* source clock for access timestamp of catcache entries */
+extern TimestampTz catcacheclock;
+
+/* SetCatCacheClock - set catcache timestamp source clodk */
+static inline void
+SetCatCacheClock(TimestampTz ts)
+{
+ catcacheclock = ts;
+}
+
extern void CreateCacheMemoryContext(void);
extern CatCache *InitCatCache(int id, Oid reloid, Oid indexoid,
--
2.16.3
----Next_Part(Fri_Mar_29_17_24_40_2019_805)--
Content-Type: Application/Octet-Stream
Content-Transfer-Encoding: base64
Content-Disposition: attachment; filename="test_script.tar.gz"
H4sIAFPSnVwAA+1b63Paxhb3V+uvOAb5WtQG9AIaO7h1J2mvZ1y74yb3fkhTRkgLqBGSol01ztjc
v/2e3ZVAflCblIhJo+MZHrtnz/t3dpElRigbUDfxY9be+kykI/V6Hf5u9Dp68T2nLcOy7Z7VMQzd
2NINq9vTt6DzuQwqUkqZkwBsTaLEH6fMX8b32PwXSqyQ/zEJB2wYtOJgvTp4grtde2n+bduQ+dc7
lmXhuNHp4hDo6zXjYfrK81/fgXZKk/bQD9sxSQJFdaMg8CmDPtRqR8ooSojjTkD1QdNbLQPT1YBr
ZXvO1uoDjRM/ZCOtdgDurt7xAL/hZ9VvHCmzokCaDilLtHzkAEzkUORqqHlJFAN1J2TqgD8CcoUs
FHiBgutQ1/HI0W8hmpTzu2gZI/kKzifn75hsPnv2TJh8Zx1zhgERy1psV7c9mNvV4HK4/dz8TSfo
M1MR/0katuhk/ToewX9XtzL8W6ah9wT+dcus8F8G1XcE9IcOnShnFxe/nJ3++qqvapS8BwNUo6H8
cHp+eXHxqt+eRFPSzqPAFykvTl6dvDi9vDvlOcwRL4MPUfJuEJlcxs8nv756edlXM3HteEzfB0WG
s8vXd2ddhOlV/O3A4PM/vj47W8Zg5gw/X7xYxmMpZyc/vDz798uTF33x6VBR3vlB4AQBxBFl44RQ
hQaExGAp6txi2RjHA5cF0GzGY+5XX808B147TCHuJAL1ei5/NnUoI0nTGOi8GYGPDRHUPLpH4EWg
MX9KgDdccFyXUMo3XgN3XriBGM2emwTH0PbIn+0wDYIGmMf/Mvj6kCxVaq6o1FyHUmtFpci/itqn
pyOKFUWwYzl9UupwXcl5kxqN1TQO/r7OcgtFalzNS3MNXq5cmWvQuJqX1mpePrG4ORDUrCt+EhB+
RMUlIyFTWS4UhNJysZCpLBcMQmm5aMhUfnY4PKXGi3jAU8KnQ2IaeZuAhVS7AWhwxRuAh1S7AYhw
xRuAiVRbClSeUP6Ilk3/MKmoFCr+/r/bOdal47Hrf1bPlL//Datrmhb+/kduvfr9Xwbdu/4nL69B
jRJ+2Y05QYRtwnEnZBAnaUgGUz8cOGMCLAJdXm/bwCXDxy7xURIQl8E3MEqiaeEa31d0Ye+JtAT/
a/0fwKPX//P//ximoRv8+l/X6tkV/sug+/ivP70B7Fm6TveqLvAF08P4Nze3/1u23P+tCv9l0D9i
/+88Bfm80irk36Ul+N/M/m/2Oj27w/Fvm90K/2XQP2j/r7rAJ9DD+Jf/n1uXjkf3f3u+/2MLyPb/
ToX/Mug+/svH8ULHH1JHQ0EFy073Kxzvt2cVwh+hZfgv8/xvm/bi/G8b1fW/EunLPP9XHWNdtAT/
5Z7/rfn1P9voCPzbRnX/Xyn0d/C/Z2zy9F/1gHVQEf9xEvEGUPb9/7bVzZ//4I+A9Pj9vz2r2v9L
ofrObfinlMAZguzw8DXzA3j/oU3TaRvh5vkEwTOJ0sBD5BLXH30E7A0w8gMCoTMlNX7TvqbWTy5/
+g/s9EFHkKp+GKdsIHj6oPKpN/pbHB/5JPACEuLot/g1dP4kCe8qfejwZhJSZxoHhOL3poHzmCPG
J/Uj5Xs38sSEhvKjGEWcnh/A3vM9Dt6FtpsbEBaPHPzm8W4leNHiFrti2KE+TDib9vz0/Fg0CHcS
TWMuczskV0z40v5dbfNOsy0cy2zgjomGIjnkrcRa65sG8orxbVVgKnCGJOBOo/04KIyRY7GTYJCl
YYdQ4K5xRqmsIGLnf9D+nStoam/05rO3DW2QfWh8J+zjKnlQ+pkqsbivmosvA+ygPGIeGfkhRkO1
4Tv+cgg1vSYlfC9uMJFR5cvyiAuRM+yJQAJ6Ow5GMQ7YdYPf6L4+Fba13u436DwgcUonmlBwwG8p
FxJzcTvoG9ZcckM/0ka+oA4JcaMEC02UAYTpdEgSiEbAJhg5P8Ftgful5OGa18vzPDvoQqGI1Lpw
b1+6k2UjjHL5oygNvax+C6KkqfdUoLi8XnNd29mm+d+Ty/PT858OcSuj6Wjkuz7B4Xylxst9Lqgh
Nk6+eKYo4hVf8iBLe3cK7NDMA56Zr7ZuFQ/msjVfuM95W9K02ly/cBOGKStIJVeIZUa8Vm7NrFgN
NEoYXKsOOn0M6nAGYuJIkUlizjsiMpKjN08QvykcU8Q1UfmYTSfLIb2fsuP74VTlrj+VRfMG9/FF
h8AwvG1Ae7FIWs3riWTL0bIIczvlt/ZLLQfwR8prhhucLXtAERdbjOAiHmpCaBqw65p6zT2bDeQ7
x9WsNrsW8Jvx5DIZG+4hdZ3ASbRxQmK4BnUA5D0IRsAwiibWKHSTHCVi4kAyFgzYvovRWyDVBWNC
vAiKXUk2h5k80LhBhAE6PceldaUOsmClW/JRKBzGcTxjpdMQmM9wJD8z4d8VprAFtRt5tssc0XI/
+MmOxrKf4xJN0xYdvgn4OmYTTfrEg2w2uGWqyw+LQna2uCXltiAb1rLxLBliwfwweAA6xilXw+cz
c1U3M3SWn2XlEbUOlGDvdViUzD1rPsGzW7wLv3DNfu2+Dlnvw8j7KMUJJGgCSO/IRwq7Mua3JO/a
FG746VM8xCYOsQ/YIXEjhkPEv3z0opZvOlmPxBrMalUIy2tz3nNJ3mHlcjzih2HE+J2FqHQK/Izv
h2N5LgYvjQOfP0ADYRQ28zmpaQ7kh9QhqAt9eG5ZgDozvX9h520wP7h8DoxZ4WQPu92WNZKBzG2c
KdvF/Hw95/qKKqqooooqqqiiiiqqqKKKKqqooooq4vR/BXY1+wBQAAA=
----Next_Part(Fri_Mar_29_17_24_40_2019_805)----
view thread (16+ messages) latest in thread
reply
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Reply to all the recipients using the --to and --cc options:
reply via email
To: [email protected]
Cc: [email protected]
Subject: Re: [PATCH] Remove entries that haven't been used for a certain time
In-Reply-To: <no-message-id-1883732@localhost>
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
This inbox is served by agora; see mirroring instructions
for how to clone and mirror all data and code used for this inbox