public inbox for [email protected]
help / color / mirror / Atom feedFrom: Kyotaro Horiguchi <[email protected]>
Subject: [PATCH 3/4] PoC add prune-by-number-of-entries feature
Date: Wed, 6 Feb 2019 14:36:29 +0900
Adds prune based on the number of cache entries on top of the current
pruning patch. It is controlled by two GUC variables.
cache_entry_limit: limit of the number of entries per catcache
cache_entry_limit_prune_ratio: how much of entries to remove at pruning
---
src/backend/utils/cache/catcache.c | 107 ++++++++++++++++++++++++++++++++++++-
src/backend/utils/misc/guc.c | 40 ++++++++++++++
src/include/utils/catcache.h | 2 +
3 files changed, 148 insertions(+), 1 deletion(-)
diff --git a/src/backend/utils/cache/catcache.c b/src/backend/utils/cache/catcache.c
index 1da1589a5d..70ae5da988 100644
--- a/src/backend/utils/cache/catcache.c
+++ b/src/backend/utils/cache/catcache.c
@@ -77,6 +77,11 @@
*/
int cache_memory_target = 0;
+
+/* PoC entry limit */
+int cache_entry_limit = 0;
+double cache_entry_limit_prune_ratio = 0.8;
+
/* GUC variable to define the minimum age of entries that will be cosidered to
* be evicted in seconds. This variable is shared among various cache
* mechanisms.
@@ -882,6 +887,102 @@ InitCatCache(int id,
return cp;
}
+/*
+ * CatCacheCleanupOldEntriesByNum -
+ * Poc remove infrequently-used entries by number of entries.
+ */
+static bool
+CatCacheCleanupOldEntriesByNum(CatCache *cp, int cache_entry_limit)
+{
+ int i;
+ int n;
+ int oldndelelem = cp->cc_ntup;
+ int ndelelem;
+ CatCTup **ct_array;
+
+ ndelelem = oldndelelem - (int)(cache_entry_limit * cache_entry_limit_prune_ratio);
+
+ /* lower limit: quite arbitrary */
+ if (ndelelem < 256)
+ ndelelem = 256;
+
+ /*
+ * partial sort array: [0] contains latest access entry
+ * [1] contains ealiest access entry
+ */
+ ct_array = (CatCTup **) palloc(ndelelem * sizeof(CatCTup*));
+ n = 0;
+
+ /*
+ * Collect entries to be removed, which have older lastaccess.
+ * Using heap bound sort like tuplesort.c.
+ */
+ for (i = 0; i < cp->cc_nbuckets; i++)
+ {
+ dlist_mutable_iter iter;
+
+ dlist_foreach_modify(iter, &cp->cc_bucket[i])
+ {
+ CatCTup *ct = dlist_container(CatCTup, cache_elem, iter.cur);
+
+ /* Don't remove referenced entries */
+ if (ct->refcount != 0 ||
+ (ct->c_list && ct->c_list->refcount != 0))
+ continue;
+
+ if (n < ndelelem)
+ {
+ /* Fill up the min heap array */
+ int j = n++;
+
+ while (j > 0)
+ {
+ int i = (j - 1) >> 1;
+
+ if (ct->lastaccess >= ct_array[i]->lastaccess)
+ break;
+ ct_array[j] = ct_array[i];
+ j = i;
+ }
+ ct_array[j] = ct;
+ }
+ else if (ct->lastaccess > ct_array[0]->lastaccess)
+ {
+ /* older than the oldest in the array, add it */
+ unsigned int i;
+
+ i = 0;
+
+ for (;;)
+ {
+ unsigned int j = 2 * i + 1;
+
+ if (j >= n)
+ break;
+ if (j + 1 < n &&
+ ct_array[j]->lastaccess > ct_array[j + 1]->lastaccess)
+ j++;
+ if (ct->lastaccess <= ct_array[j]->lastaccess)
+ break;
+ ct_array[i] = ct_array[j];
+ i = j;
+ }
+ ct_array[i] = ct;
+ }
+ }
+ }
+
+ /* Now we have the list of elements to be deleted */
+ for (i = 0 ; i < n && ct_array[i]; i++)
+ CatCacheRemoveCTup(cp, ct_array[i]);
+
+ pfree(ct_array);
+
+ elog(LOG, "Catcache pruned by entry number: id=%d, %d => %d", cp->id, oldndelelem, cp->cc_ntup);
+
+ return true;
+}
+
/*
* CatCacheCleanupOldEntries - Remove infrequently-used entries
*
@@ -923,7 +1024,7 @@ CatCacheCleanupOldEntries(CatCache *cp)
hash_size = cp->cc_nbuckets * sizeof(dlist_head);
if (hash_size + cp->cc_tupsize < (Size) cache_memory_target * 1024L)
return false;
-
+
/*
* Search the whole hash for entries to remove. This is a quite time
* consuming task during catcache lookup, but accetable since now we are
@@ -2047,6 +2148,10 @@ CatalogCacheCreateEntry(CatCache *cache, HeapTuple ntp, Datum *arguments,
CacheHdr->ch_ntup++;
cache->cc_tupsize += tupsize;
+ /* cap number of entries */
+ if (cache_entry_limit > 0 && cache->cc_ntup > cache_entry_limit)
+ CatCacheCleanupOldEntriesByNum(cache, cache_entry_limit);
+
/*
* If the hash table has become too full, try cleanup by removing
* infrequently used entries to make a room for the new entry. If it
diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c
index 32e41253a6..7bb239a07e 100644
--- a/src/backend/utils/misc/guc.c
+++ b/src/backend/utils/misc/guc.c
@@ -2227,6 +2227,36 @@ static struct config_int ConfigureNamesInt[] =
NULL, NULL, NULL
},
+ {
+ {"cache_entry_limit", PGC_USERSET, RESOURCES_MEM,
+ gettext_noop("Sets the maximum entries of catcache."),
+ NULL
+ },
+ &cache_entry_limit,
+ 0, 0, INT_MAX,
+ NULL, NULL, NULL
+ },
+
+ {
+ {"cache_entry_limit", PGC_USERSET, RESOURCES_MEM,
+ gettext_noop("Sets the maximum entries of catcache."),
+ NULL
+ },
+ &cache_entry_limit,
+ 0, 0, INT_MAX,
+ NULL, NULL, NULL
+ },
+
+ {
+ {"cache_entry_limit", PGC_USERSET, RESOURCES_MEM,
+ gettext_noop("Sets the maximum entries of catcache."),
+ NULL
+ },
+ &cache_entry_limit,
+ 0, 0, INT_MAX,
+ NULL, NULL, NULL
+ },
+
/*
* We use the hopefully-safely-small value of 100kB as the compiled-in
* default for max_stack_depth. InitializeGUCOptions will increase it if
@@ -3401,6 +3431,16 @@ static struct config_real ConfigureNamesReal[] =
NULL, NULL, NULL
},
+ {
+ {"cache_entry_limit_prune_ratio", PGC_USERSET, RESOURCES_MEM,
+ gettext_noop("Sets the maximum entries of catcache."),
+ NULL
+ },
+ &cache_entry_limit_prune_ratio,
+ 0.8, 0.0, 1.0,
+ NULL, NULL, NULL
+ },
+
/* End-of-list marker */
{
{NULL, 0, 0, NULL, NULL}, NULL, 0.0, 0.0, 0.0, NULL, NULL, NULL
diff --git a/src/include/utils/catcache.h b/src/include/utils/catcache.h
index 4d51975920..1f7fb51ac0 100644
--- a/src/include/utils/catcache.h
+++ b/src/include/utils/catcache.h
@@ -193,6 +193,8 @@ extern PGDLLIMPORT MemoryContext CacheMemoryContext;
/* for guc.c, not PGDLLPMPORT'ed */
extern int cache_prune_min_age;
extern int cache_memory_target;
+extern int cache_entry_limit;
+extern double cache_entry_limit_prune_ratio;
/* to use as access timestamp of catcache entries */
extern TimestampTz catcacheclock;
--
2.16.3
----Next_Part(Wed_Feb_06_15_17_59_2019_321)--
Content-Type: Text/Plain; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline; filename="gen.pl"
#! /usr/bin/perl
print "set track_syscache_usage_interval to 1000;\n";
## for time-based pruning
#print "set cache_prune_min_age to '5s';\n";
#print "set cache_memory_target to '0';\n";
## for limit-based pruning
print "set cache_memory_target to '100MB';\n";
print "set cache_entry_limit to 10000;\n";
print "set cache_entry_limit_prune_ratio to 0.6;\n";
while (1) {
print "begin; create temp table t1 (a int, b int, c int, d int, e int, f int, g int, h int, i int, j int) on commit drop; insert into t1 values (1, 2, 3, 4, 5, 6, 7, 8, 9, 10); select * from t1; commit;\n";
}
----Next_Part(Wed_Feb_06_15_17_59_2019_321)----
view thread (12+ messages) latest in thread
reply
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Reply to all the recipients using the --to and --cc options:
reply via email
To: [email protected]
Cc: [email protected]
Subject: Re: [PATCH 3/4] PoC add prune-by-number-of-entries feature
In-Reply-To: <no-message-id-1883439@localhost>
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
This inbox is served by agora; see mirroring instructions
for how to clone and mirror all data and code used for this inbox