From: Kyotaro Horiguchi Date: Wed, 6 Feb 2019 14:36:29 +0900 Subject: [PATCH 3/4] PoC add prune-by-number-of-entries feature Adds prune based on the number of cache entries on top of the current pruning patch. It is controlled by two GUC variables. cache_entry_limit: limit of the number of entries per catcache cache_entry_limit_prune_ratio: how much of entries to remove at pruning --- src/backend/utils/cache/catcache.c | 107 ++++++++++++++++++++++++++++++++++++- src/backend/utils/misc/guc.c | 40 ++++++++++++++ src/include/utils/catcache.h | 2 + 3 files changed, 148 insertions(+), 1 deletion(-) diff --git a/src/backend/utils/cache/catcache.c b/src/backend/utils/cache/catcache.c index 1da1589a5d..70ae5da988 100644 --- a/src/backend/utils/cache/catcache.c +++ b/src/backend/utils/cache/catcache.c @@ -77,6 +77,11 @@ */ int cache_memory_target = 0; + +/* PoC entry limit */ +int cache_entry_limit = 0; +double cache_entry_limit_prune_ratio = 0.8; + /* GUC variable to define the minimum age of entries that will be cosidered to * be evicted in seconds. This variable is shared among various cache * mechanisms. @@ -882,6 +887,102 @@ InitCatCache(int id, return cp; } +/* + * CatCacheCleanupOldEntriesByNum - + * Poc remove infrequently-used entries by number of entries. + */ +static bool +CatCacheCleanupOldEntriesByNum(CatCache *cp, int cache_entry_limit) +{ + int i; + int n; + int oldndelelem = cp->cc_ntup; + int ndelelem; + CatCTup **ct_array; + + ndelelem = oldndelelem - (int)(cache_entry_limit * cache_entry_limit_prune_ratio); + + /* lower limit: quite arbitrary */ + if (ndelelem < 256) + ndelelem = 256; + + /* + * partial sort array: [0] contains latest access entry + * [1] contains ealiest access entry + */ + ct_array = (CatCTup **) palloc(ndelelem * sizeof(CatCTup*)); + n = 0; + + /* + * Collect entries to be removed, which have older lastaccess. + * Using heap bound sort like tuplesort.c. + */ + for (i = 0; i < cp->cc_nbuckets; i++) + { + dlist_mutable_iter iter; + + dlist_foreach_modify(iter, &cp->cc_bucket[i]) + { + CatCTup *ct = dlist_container(CatCTup, cache_elem, iter.cur); + + /* Don't remove referenced entries */ + if (ct->refcount != 0 || + (ct->c_list && ct->c_list->refcount != 0)) + continue; + + if (n < ndelelem) + { + /* Fill up the min heap array */ + int j = n++; + + while (j > 0) + { + int i = (j - 1) >> 1; + + if (ct->lastaccess >= ct_array[i]->lastaccess) + break; + ct_array[j] = ct_array[i]; + j = i; + } + ct_array[j] = ct; + } + else if (ct->lastaccess > ct_array[0]->lastaccess) + { + /* older than the oldest in the array, add it */ + unsigned int i; + + i = 0; + + for (;;) + { + unsigned int j = 2 * i + 1; + + if (j >= n) + break; + if (j + 1 < n && + ct_array[j]->lastaccess > ct_array[j + 1]->lastaccess) + j++; + if (ct->lastaccess <= ct_array[j]->lastaccess) + break; + ct_array[i] = ct_array[j]; + i = j; + } + ct_array[i] = ct; + } + } + } + + /* Now we have the list of elements to be deleted */ + for (i = 0 ; i < n && ct_array[i]; i++) + CatCacheRemoveCTup(cp, ct_array[i]); + + pfree(ct_array); + + elog(LOG, "Catcache pruned by entry number: id=%d, %d => %d", cp->id, oldndelelem, cp->cc_ntup); + + return true; +} + /* * CatCacheCleanupOldEntries - Remove infrequently-used entries * @@ -923,7 +1024,7 @@ CatCacheCleanupOldEntries(CatCache *cp) hash_size = cp->cc_nbuckets * sizeof(dlist_head); if (hash_size + cp->cc_tupsize < (Size) cache_memory_target * 1024L) return false; - + /* * Search the whole hash for entries to remove. This is a quite time * consuming task during catcache lookup, but accetable since now we are @@ -2047,6 +2148,10 @@ CatalogCacheCreateEntry(CatCache *cache, HeapTuple ntp, Datum *arguments, CacheHdr->ch_ntup++; cache->cc_tupsize += tupsize; + /* cap number of entries */ + if (cache_entry_limit > 0 && cache->cc_ntup > cache_entry_limit) + CatCacheCleanupOldEntriesByNum(cache, cache_entry_limit); + /* * If the hash table has become too full, try cleanup by removing * infrequently used entries to make a room for the new entry. If it diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c index 32e41253a6..7bb239a07e 100644 --- a/src/backend/utils/misc/guc.c +++ b/src/backend/utils/misc/guc.c @@ -2227,6 +2227,36 @@ static struct config_int ConfigureNamesInt[] = NULL, NULL, NULL }, + { + {"cache_entry_limit", PGC_USERSET, RESOURCES_MEM, + gettext_noop("Sets the maximum entries of catcache."), + NULL + }, + &cache_entry_limit, + 0, 0, INT_MAX, + NULL, NULL, NULL + }, + + { + {"cache_entry_limit", PGC_USERSET, RESOURCES_MEM, + gettext_noop("Sets the maximum entries of catcache."), + NULL + }, + &cache_entry_limit, + 0, 0, INT_MAX, + NULL, NULL, NULL + }, + + { + {"cache_entry_limit", PGC_USERSET, RESOURCES_MEM, + gettext_noop("Sets the maximum entries of catcache."), + NULL + }, + &cache_entry_limit, + 0, 0, INT_MAX, + NULL, NULL, NULL + }, + /* * We use the hopefully-safely-small value of 100kB as the compiled-in * default for max_stack_depth. InitializeGUCOptions will increase it if @@ -3401,6 +3431,16 @@ static struct config_real ConfigureNamesReal[] = NULL, NULL, NULL }, + { + {"cache_entry_limit_prune_ratio", PGC_USERSET, RESOURCES_MEM, + gettext_noop("Sets the maximum entries of catcache."), + NULL + }, + &cache_entry_limit_prune_ratio, + 0.8, 0.0, 1.0, + NULL, NULL, NULL + }, + /* End-of-list marker */ { {NULL, 0, 0, NULL, NULL}, NULL, 0.0, 0.0, 0.0, NULL, NULL, NULL diff --git a/src/include/utils/catcache.h b/src/include/utils/catcache.h index 4d51975920..1f7fb51ac0 100644 --- a/src/include/utils/catcache.h +++ b/src/include/utils/catcache.h @@ -193,6 +193,8 @@ extern PGDLLIMPORT MemoryContext CacheMemoryContext; /* for guc.c, not PGDLLPMPORT'ed */ extern int cache_prune_min_age; extern int cache_memory_target; +extern int cache_entry_limit; +extern double cache_entry_limit_prune_ratio; /* to use as access timestamp of catcache entries */ extern TimestampTz catcacheclock; -- 2.16.3 ----Next_Part(Wed_Feb_06_15_17_59_2019_321)-- Content-Type: Text/Plain; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="gen.pl" #! /usr/bin/perl print "set track_syscache_usage_interval to 1000;\n"; ## for time-based pruning #print "set cache_prune_min_age to '5s';\n"; #print "set cache_memory_target to '0';\n"; ## for limit-based pruning print "set cache_memory_target to '100MB';\n"; print "set cache_entry_limit to 10000;\n"; print "set cache_entry_limit_prune_ratio to 0.6;\n"; while (1) { print "begin; create temp table t1 (a int, b int, c int, d int, e int, f int, g int, h int, i int, j int) on commit drop; insert into t1 values (1, 2, 3, 4, 5, 6, 7, 8, 9, 10); select * from t1; commit;\n"; } ----Next_Part(Wed_Feb_06_15_17_59_2019_321)----