agora inbox for [email protected]
help / color / mirror / Atom feed[PATCH 1/4] Remove entries that haven't been used for a certain time
9+ messages / 5 participants
[nested] [flat]
* [PATCH 1/4] Remove entries that haven't been used for a certain time
@ 2017-12-26 08:43 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 9+ messages in thread
From: Kyotaro Horiguchi @ 2017-12-26 08:43 UTC (permalink / raw)
Catcache entries can be left alone for several reasons. It is not
desirable that they eat up memory. With this patch, This adds
consideration of removal of entries that haven't been used for a
certain time before enlarging the hash array.
---
doc/src/sgml/config.sgml | 38 ++++++
src/backend/access/transam/xact.c | 3 +
src/backend/utils/cache/catcache.c | 153 +++++++++++++++++++++++-
src/backend/utils/cache/plancache.c | 163 ++++++++++++++++++++++++++
src/backend/utils/misc/guc.c | 33 ++++++
src/backend/utils/misc/postgresql.conf.sample | 2 +
src/include/utils/catcache.h | 19 +++
src/include/utils/plancache.h | 7 +-
8 files changed, 413 insertions(+), 5 deletions(-)
diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index 5b913f00c1..76745047af 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -1617,6 +1617,44 @@ include_dir 'conf.d'
</listitem>
</varlistentry>
+ <varlistentry id="guc-syscache-memory-target" xreflabel="syscache_memory_target">
+ <term><varname>syscache_memory_target</varname> (<type>integer</type>)
+ <indexterm>
+ <primary><varname>syscache_memory_target</varname> configuration parameter</primary>
+ </indexterm>
+ </term>
+ <listitem>
+ <para>
+ Specifies the maximum amount of memory to which syscache is expanded
+ without pruning. The value defaults to 0, indicating that pruning is
+ always considered. After exceeding this size, syscache pruning is
+ considered according to
+ <xref linkend="guc-syscache-prune-min-age"/>. If you need to keep
+ certain amount of syscache entries with intermittent usage, try
+ increase this setting.
+ </para>
+ </listitem>
+ </varlistentry>
+
+ <varlistentry id="guc-syscache-prune-min-age" xreflabel="syscache_prune_min_age">
+ <term><varname>syscache_prune_min_age</varname> (<type>integer</type>)
+ <indexterm>
+ <primary><varname>syscache_prune_min_age</varname> configuration parameter</primary>
+ </indexterm>
+ </term>
+ <listitem>
+ <para>
+ Specifies the minimum amount of unused time in seconds at which a
+ syscache entry is considered to be removed. -1 indicates that syscache
+ pruning is disabled at all. The value defaults to 600 seconds
+ (<literal>10 minutes</literal>). The syscache entries that are not
+ used for the duration can be removed to prevent syscache bloat. This
+ behavior is suppressed until the size of syscache exceeds
+ <xref linkend="guc-syscache-memory-target"/>.
+ </para>
+ </listitem>
+ </varlistentry>
+
<varlistentry id="guc-max-stack-depth" xreflabel="max_stack_depth">
<term><varname>max_stack_depth</varname> (<type>integer</type>)
<indexterm>
diff --git a/src/backend/access/transam/xact.c b/src/backend/access/transam/xact.c
index 8e6aef332c..e4a4a5874c 100644
--- a/src/backend/access/transam/xact.c
+++ b/src/backend/access/transam/xact.c
@@ -732,6 +732,9 @@ void
SetCurrentStatementStartTimestamp(void)
{
stmtStartTimestamp = GetCurrentTimestamp();
+
+ /* Set this timestamp as aproximated current time */
+ SetCatCacheClock(stmtStartTimestamp);
}
/*
diff --git a/src/backend/utils/cache/catcache.c b/src/backend/utils/cache/catcache.c
index 5ddbf6eab1..9f421cd242 100644
--- a/src/backend/utils/cache/catcache.c
+++ b/src/backend/utils/cache/catcache.c
@@ -71,9 +71,24 @@
#define CACHE6_elog(a,b,c,d,e,f,g)
#endif
+/*
+ * GUC variable to define the minimum size of hash to cosider entry eviction.
+ * This variable is shared among various cache mechanisms.
+ */
+int cache_memory_target = 0;
+
+/* GUC variable to define the minimum age of entries that will be cosidered to
+ * be evicted in seconds. This variable is shared among various cache
+ * mechanisms.
+ */
+int cache_prune_min_age = 600;
+
/* Cache management header --- pointer is NULL until created */
static CatCacheHeader *CacheHdr = NULL;
+/* Timestamp used for any operation on caches. */
+TimestampTz catcacheclock = 0;
+
static inline HeapTuple SearchCatCacheInternal(CatCache *cache,
int nkeys,
Datum v1, Datum v2,
@@ -866,9 +881,130 @@ InitCatCache(int id,
*/
MemoryContextSwitchTo(oldcxt);
+ /* initilize catcache reference clock if haven't done yet */
+ if (catcacheclock == 0)
+ catcacheclock = GetCurrentTimestamp();
+
return cp;
}
+/*
+ * CatCacheCleanupOldEntries - Remove infrequently-used entries
+ *
+ * Catcache entries can be left alone for several reasons. We remove them if
+ * they are not accessed for a certain time to prevent catcache from
+ * bloating. The eviction is performed with the similar algorithm with buffer
+ * eviction using access counter. Entries that are accessed several times can
+ * live longer than those that have had no access in the same duration.
+ */
+static bool
+CatCacheCleanupOldEntries(CatCache *cp)
+{
+ int i;
+ int nremoved = 0;
+ size_t hash_size;
+#ifdef CATCACHE_STATS
+ /* These variables are only for debugging purpose */
+ int ntotal = 0;
+ /*
+ * nth element in nentries stores the number of cache entries that have
+ * lived unaccessed for corresponding multiple in ageclass of
+ * cache_prune_min_age. The index of nremoved_entry is the value of the
+ * clock-sweep counter, which takes from 0 up to 2.
+ */
+ double ageclass[] = {0.05, 0.1, 1.0, 2.0, 3.0, 0.0};
+ int nentries[] = {0, 0, 0, 0, 0, 0};
+ int nremoved_entry[3] = {0, 0, 0};
+ int j;
+#endif
+
+ /* Return immediately if no pruning is wanted */
+ if (cache_prune_min_age < 0)
+ return false;
+
+ /*
+ * Return without pruning if the size of the hash is below the target.
+ * Since the area for bucket array is dominant, consider only it.
+ */
+ hash_size = cp->cc_nbuckets * sizeof(dlist_head);
+ if (hash_size < (Size) cache_memory_target * 1024L)
+ return false;
+
+ /* Search the whole hash for entries to remove */
+ for (i = 0; i < cp->cc_nbuckets; i++)
+ {
+ dlist_mutable_iter iter;
+
+ dlist_foreach_modify(iter, &cp->cc_bucket[i])
+ {
+ CatCTup *ct = dlist_container(CatCTup, cache_elem, iter.cur);
+ long entry_age;
+ int us;
+
+
+ /*
+ * Calculate the duration from the time of the last access to the
+ * "current" time. Since catcacheclock is not advanced within a
+ * transaction, the entries that are accessed within the current
+ * transaction won't be pruned.
+ */
+ TimestampDifference(ct->lastaccess, catcacheclock, &entry_age, &us);
+
+#ifdef CATCACHE_STATS
+ /* count catcache entries for each age class */
+ ntotal++;
+ for (j = 0 ;
+ ageclass[j] != 0.0 &&
+ entry_age > cache_prune_min_age * ageclass[j] ;
+ j++);
+ if (ageclass[j] == 0.0) j--;
+ nentries[j]++;
+#endif
+
+ /*
+ * Try to remove entries older than cache_prune_min_age seconds.
+ * Entries that are not accessed after last pruning are removed in
+ * that seconds, and that has been accessed several times are
+ * removed after leaving alone for up to three times of the
+ * duration. We don't try shrink buckets since pruning effectively
+ * caps catcache expansion in the long term.
+ */
+ if (entry_age > cache_prune_min_age)
+ {
+#ifdef CATCACHE_STATS
+ Assert (ct->naccess >= 0 && ct->naccess <= 2);
+ nremoved_entry[ct->naccess]++;
+#endif
+ if (ct->naccess > 0)
+ ct->naccess--;
+ else
+ {
+ if (!ct->c_list || ct->c_list->refcount == 0)
+ {
+ CatCacheRemoveCTup(cp, ct);
+ nremoved++;
+ }
+ }
+ }
+ }
+ }
+
+#ifdef CATCACHE_STATS
+ ereport(DEBUG1,
+ (errmsg ("removed %d/%d, age(-%.0fs:%d, -%.0fs:%d, *-%.0fs:%d, -%.0fs:%d, -%.0fs:%d) naccessed(0:%d, 1:%d, 2:%d)",
+ nremoved, ntotal,
+ ageclass[0] * cache_prune_min_age, nentries[0],
+ ageclass[1] * cache_prune_min_age, nentries[1],
+ ageclass[2] * cache_prune_min_age, nentries[2],
+ ageclass[3] * cache_prune_min_age, nentries[3],
+ ageclass[4] * cache_prune_min_age, nentries[4],
+ nremoved_entry[0], nremoved_entry[1], nremoved_entry[2]),
+ errhidestmt(true)));
+#endif
+
+ return nremoved > 0;
+}
+
/*
* Enlarge a catcache, doubling the number of buckets.
*/
@@ -1282,6 +1418,11 @@ SearchCatCacheInternal(CatCache *cache,
*/
dlist_move_head(bucket, &ct->cache_elem);
+ /* Update access information for pruning */
+ if (ct->naccess < 2)
+ ct->naccess++;
+ ct->lastaccess = catcacheclock;
+
/*
* If it's a positive entry, bump its refcount and return it. If it's
* negative, we can report failure to the caller.
@@ -1813,7 +1954,6 @@ ReleaseCatCacheList(CatCList *list)
CatCacheRemoveCList(list->my_cache, list);
}
-
/*
* CatalogCacheCreateEntry
* Create a new CatCTup entry, copying the given HeapTuple and other
@@ -1906,6 +2046,8 @@ CatalogCacheCreateEntry(CatCache *cache, HeapTuple ntp, Datum *arguments,
ct->dead = false;
ct->negative = negative;
ct->hash_value = hashValue;
+ ct->naccess = 0;
+ ct->lastaccess = catcacheclock;
dlist_push_head(&cache->cc_bucket[hashIndex], &ct->cache_elem);
@@ -1913,10 +2055,13 @@ CatalogCacheCreateEntry(CatCache *cache, HeapTuple ntp, Datum *arguments,
CacheHdr->ch_ntup++;
/*
- * If the hash table has become too full, enlarge the buckets array. Quite
- * arbitrarily, we enlarge when fill factor > 2.
+ * If the hash table has become too full, try cleanup by removing
+ * infrequently used entries to make a room for the new entry. If it
+ * failed, enlarge the bucket array instead. Quite arbitrarily, we try
+ * this when fill factor > 2.
*/
- if (cache->cc_ntup > cache->cc_nbuckets * 2)
+ if (cache->cc_ntup > cache->cc_nbuckets * 2 &&
+ !CatCacheCleanupOldEntries(cache))
RehashCatCache(cache);
return ct;
diff --git a/src/backend/utils/cache/plancache.c b/src/backend/utils/cache/plancache.c
index 0ad3e3c736..701ead152c 100644
--- a/src/backend/utils/cache/plancache.c
+++ b/src/backend/utils/cache/plancache.c
@@ -63,12 +63,14 @@
#include "storage/lmgr.h"
#include "tcop/pquery.h"
#include "tcop/utility.h"
+#include "utils/catcache.h"
#include "utils/inval.h"
#include "utils/memutils.h"
#include "utils/resowner_private.h"
#include "utils/rls.h"
#include "utils/snapmgr.h"
#include "utils/syscache.h"
+#include "utils/timestamp.h"
/*
@@ -86,6 +88,12 @@
* guarantee to save a CachedPlanSource without error.
*/
static CachedPlanSource *first_saved_plan = NULL;
+static CachedPlanSource *last_saved_plan = NULL;
+static int num_saved_plans = 0;
+static TimestampTz oldest_saved_plan = 0;
+
+/* GUC variables */
+int min_cached_plans = 1000;
static void ReleaseGenericPlan(CachedPlanSource *plansource);
static List *RevalidateCachedQuery(CachedPlanSource *plansource,
@@ -105,6 +113,7 @@ static TupleDesc PlanCacheComputeResultDesc(List *stmt_list);
static void PlanCacheRelCallback(Datum arg, Oid relid);
static void PlanCacheFuncCallback(Datum arg, int cacheid, uint32 hashvalue);
static void PlanCacheSysCallback(Datum arg, int cacheid, uint32 hashvalue);
+static void PruneCachedPlan(void);
/*
@@ -208,6 +217,8 @@ CreateCachedPlan(RawStmt *raw_parse_tree,
plansource->generic_cost = -1;
plansource->total_custom_cost = 0;
plansource->num_custom_plans = 0;
+ plansource->last_access = GetCatCacheClock();
+
MemoryContextSwitchTo(oldcxt);
@@ -423,6 +434,28 @@ CompleteCachedPlan(CachedPlanSource *plansource,
plansource->is_valid = true;
}
+/* moves the plansource to the first in the list */
+static inline void
+MovePlansourceToFirst(CachedPlanSource *plansource)
+{
+ if (first_saved_plan != plansource)
+ {
+ /* delink this element */
+ if (plansource->next_saved)
+ plansource->next_saved->prev_saved = plansource->prev_saved;
+ if (plansource->prev_saved)
+ plansource->prev_saved->next_saved = plansource->next_saved;
+ if (last_saved_plan == plansource)
+ last_saved_plan = plansource->prev_saved;
+
+ /* insert at the beginning */
+ first_saved_plan->prev_saved = plansource;
+ plansource->next_saved = first_saved_plan;
+ plansource->prev_saved = NULL;
+ first_saved_plan = plansource;
+ }
+}
+
/*
* SaveCachedPlan: save a cached plan permanently
*
@@ -470,6 +503,11 @@ SaveCachedPlan(CachedPlanSource *plansource)
* Add the entry to the global list of cached plans.
*/
plansource->next_saved = first_saved_plan;
+ if (first_saved_plan)
+ first_saved_plan->prev_saved = plansource;
+ else
+ last_saved_plan = plansource;
+ plansource->prev_saved = NULL;
first_saved_plan = plansource;
plansource->is_saved = true;
@@ -492,7 +530,11 @@ DropCachedPlan(CachedPlanSource *plansource)
if (plansource->is_saved)
{
if (first_saved_plan == plansource)
+ {
first_saved_plan = plansource->next_saved;
+ if (first_saved_plan)
+ first_saved_plan->prev_saved = NULL;
+ }
else
{
CachedPlanSource *psrc;
@@ -502,10 +544,19 @@ DropCachedPlan(CachedPlanSource *plansource)
if (psrc->next_saved == plansource)
{
psrc->next_saved = plansource->next_saved;
+ if (psrc->next_saved)
+ psrc->next_saved->prev_saved = psrc;
break;
}
}
}
+
+ if (last_saved_plan == plansource)
+ {
+ last_saved_plan = plansource->prev_saved;
+ if (last_saved_plan)
+ last_saved_plan->next_saved = NULL;
+ }
plansource->is_saved = false;
}
@@ -537,6 +588,13 @@ ReleaseGenericPlan(CachedPlanSource *plansource)
Assert(plan->magic == CACHEDPLAN_MAGIC);
plansource->gplan = NULL;
ReleaseCachedPlan(plan, false);
+
+ /* decrement "saved plans" counter */
+ if (plansource->is_saved)
+ {
+ Assert (num_saved_plans > 0);
+ num_saved_plans--;
+ }
}
}
@@ -1148,6 +1206,17 @@ GetCachedPlan(CachedPlanSource *plansource, ParamListInfo boundParams,
if (useResOwner && !plansource->is_saved)
elog(ERROR, "cannot apply ResourceOwner to non-saved cached plan");
+ /*
+ * set last-accessed timestamp and move this plan to the first of the list
+ */
+ if (plansource->is_saved)
+ {
+ plansource->last_access = GetCatCacheClock();
+
+ /* move this plan to the first of the list */
+ MovePlansourceToFirst(plansource);
+ }
+
/* Make sure the querytree list is valid and we have parse-time locks */
qlist = RevalidateCachedQuery(plansource, queryEnv);
@@ -1156,6 +1225,11 @@ GetCachedPlan(CachedPlanSource *plansource, ParamListInfo boundParams,
if (!customplan)
{
+ /* Prune cached plans if needed */
+ if (plansource->is_saved &&
+ min_cached_plans >= 0 && num_saved_plans > min_cached_plans)
+ PruneCachedPlan();
+
if (CheckCachedPlan(plansource))
{
/* We want a generic plan, and we already have a valid one */
@@ -1168,6 +1242,11 @@ GetCachedPlan(CachedPlanSource *plansource, ParamListInfo boundParams,
plan = BuildCachedPlan(plansource, qlist, NULL, queryEnv);
/* Just make real sure plansource->gplan is clear */
ReleaseGenericPlan(plansource);
+
+ /* count this new saved plan */
+ if (plansource->is_saved)
+ num_saved_plans++;
+
/* Link the new generic plan into the plansource */
plansource->gplan = plan;
plan->refcount++;
@@ -1856,6 +1935,90 @@ PlanCacheSysCallback(Datum arg, int cacheid, uint32 hashvalue)
ResetPlanCache();
}
+/*
+ * PrunePlanCache: removes generic plan of "old" saved plans.
+ */
+static void
+PruneCachedPlan(void)
+{
+ CachedPlanSource *plansource;
+ TimestampTz currclock = GetCatCacheClock();
+ long age;
+ int us;
+ int nremoved = 0;
+
+ /* do nothing if not wanted */
+ if (cache_prune_min_age < 0 || num_saved_plans <= min_cached_plans)
+ return;
+
+ /* Fast check for oldest cache */
+ if (oldest_saved_plan > 0)
+ {
+ TimestampDifference(oldest_saved_plan, currclock, &age, &us);
+ if (age < cache_prune_min_age)
+ return;
+ }
+
+ /* last plan is the oldest. */
+ for (plansource = last_saved_plan; plansource; plansource = plansource->prev_saved)
+ {
+ long plan_age;
+ int us;
+
+ Assert(plansource->magic == CACHEDPLANSOURCE_MAGIC);
+
+ /* we want to prune no more plans */
+ if (num_saved_plans <= min_cached_plans)
+ break;
+
+ /*
+ * No work if it already doesn't have gplan and move it to the
+ * beginning so that we don't see it at the next time
+ */
+ if (!plansource->gplan)
+ continue;
+
+ /*
+ * Check age for pruning. Can exit immediately when finding a
+ * not-older element.
+ */
+ TimestampDifference(plansource->last_access, currclock, &plan_age, &us);
+ if (plan_age <= cache_prune_min_age)
+ {
+ /* this entry is the next oldest */
+ oldest_saved_plan = plansource->last_access;
+ break;
+ }
+
+ /*
+ * Here, remove generic plans of this plansrouceif it is not actually
+ * used and move it to the beginning of the list. Just update
+ * last_access and move it to the beginning if the plan is used.
+ */
+ if (plansource->gplan->refcount <= 1)
+ {
+ ReleaseGenericPlan(plansource);
+ nremoved++;
+ }
+
+ plansource->last_access = currclock;
+ }
+
+ /* move the "removed" plansrouces altogehter to the beginning of the list */
+ if (plansource != last_saved_plan && plansource)
+ {
+ plansource->next_saved->prev_saved = NULL;
+ first_saved_plan->prev_saved = last_saved_plan;
+ last_saved_plan->next_saved = first_saved_plan;
+ first_saved_plan = plansource->next_saved;
+ plansource->next_saved = NULL;
+ last_saved_plan = plansource;
+ }
+
+ if (nremoved > 0)
+ elog(DEBUG1, "plancache removed %d/%d", nremoved, num_saved_plans);
+}
+
/*
* ResetPlanCache: invalidate all cached plans.
*/
diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c
index b05fb209bb..e49346707d 100644
--- a/src/backend/utils/misc/guc.c
+++ b/src/backend/utils/misc/guc.c
@@ -79,6 +79,7 @@
#include "tsearch/ts_cache.h"
#include "utils/builtins.h"
#include "utils/bytea.h"
+#include "utils/catcache.h"
#include "utils/guc_tables.h"
#include "utils/memutils.h"
#include "utils/pg_locale.h"
@@ -2105,6 +2106,38 @@ static struct config_int ConfigureNamesInt[] =
NULL, NULL, NULL
},
+ {
+ {"cache_memory_target", PGC_USERSET, RESOURCES_MEM,
+ gettext_noop("Sets the minimum syscache size to keep."),
+ gettext_noop("Cache is not pruned before exceeding this size."),
+ GUC_UNIT_KB
+ },
+ &cache_memory_target,
+ 0, 0, MAX_KILOBYTES,
+ NULL, NULL, NULL
+ },
+
+ {
+ {"cache_prune_min_age", PGC_USERSET, RESOURCES_MEM,
+ gettext_noop("Sets the minimum unused duration of cache entries before removal."),
+ gettext_noop("Cache entries that live unused for longer than this seconds are considered to be removed."),
+ GUC_UNIT_S
+ },
+ &cache_prune_min_age,
+ 600, -1, INT_MAX,
+ NULL, NULL, NULL
+ },
+
+ {
+ {"min_cached_plans", PGC_USERSET, RESOURCES_MEM,
+ gettext_noop("Sets the minimum number of cached plans kept on memory."),
+ gettext_noop("Timeout invalidation of plancache is not activated until the number of plancaches reaches this value. -1 means timeout invalidation is always active.")
+ },
+ &min_cached_plans,
+ 1000, -1, INT_MAX,
+ NULL, NULL, NULL
+ },
+
/*
* We use the hopefully-safely-small value of 100kB as the compiled-in
* default for max_stack_depth. InitializeGUCOptions will increase it if
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index 9e39baf466..3f2760ef9d 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -126,6 +126,8 @@
#work_mem = 4MB # min 64kB
#maintenance_work_mem = 64MB # min 1MB
#autovacuum_work_mem = -1 # min 1MB, or -1 to use maintenance_work_mem
+#cache_memory_target = 0kB # in kB
+#cache_prune_min_age = 600s # -1 disables pruning
#max_stack_depth = 2MB # min 100kB
#dynamic_shared_memory_type = posix # the default is the first option
# supported by the operating system:
diff --git a/src/include/utils/catcache.h b/src/include/utils/catcache.h
index 7b22f9c7bc..599303be56 100644
--- a/src/include/utils/catcache.h
+++ b/src/include/utils/catcache.h
@@ -22,6 +22,7 @@
#include "access/htup.h"
#include "access/skey.h"
+#include "datatype/timestamp.h"
#include "lib/ilist.h"
#include "utils/relcache.h"
@@ -119,6 +120,8 @@ typedef struct catctup
bool dead; /* dead but not yet removed? */
bool negative; /* negative cache entry? */
HeapTupleData tuple; /* tuple management header */
+ int naccess; /* # of access to this entry, up to 2 */
+ TimestampTz lastaccess; /* approx. timestamp of the last usage */
/*
* The tuple may also be a member of at most one CatCList. (If a single
@@ -189,6 +192,22 @@ typedef struct catcacheheader
/* this extern duplicates utils/memutils.h... */
extern PGDLLIMPORT MemoryContext CacheMemoryContext;
+/* for guc.c, not PGDLLPMPORT'ed */
+extern int cache_prune_min_age;
+extern int cache_memory_target;
+
+/* to use as access timestamp of catcache entries */
+extern TimestampTz catcacheclock;
+
+/*
+ * SetCatCacheClock - set timestamp for catcache access record
+ */
+static inline void
+SetCatCacheClock(TimestampTz ts)
+{
+ catcacheclock = ts;
+}
+
extern void CreateCacheMemoryContext(void);
extern CatCache *InitCatCache(int id, Oid reloid, Oid indexoid,
diff --git a/src/include/utils/plancache.h b/src/include/utils/plancache.h
index ab20aa04b0..f3c5b2010d 100644
--- a/src/include/utils/plancache.h
+++ b/src/include/utils/plancache.h
@@ -110,11 +110,13 @@ typedef struct CachedPlanSource
bool is_valid; /* is the query_list currently valid? */
int generation; /* increments each time we create a plan */
/* If CachedPlanSource has been saved, it is a member of a global list */
- struct CachedPlanSource *next_saved; /* list link, if so */
+ struct CachedPlanSource *prev_saved; /* list prev link, if so */
+ struct CachedPlanSource *next_saved; /* list next link, if so */
/* State kept to help decide whether to use custom or generic plans: */
double generic_cost; /* cost of generic plan, or -1 if not known */
double total_custom_cost; /* total cost of custom plans so far */
int num_custom_plans; /* number of plans included in total */
+ TimestampTz last_access; /* timestamp of the last usage */
} CachedPlanSource;
/*
@@ -143,6 +145,9 @@ typedef struct CachedPlan
MemoryContext context; /* context containing this CachedPlan */
} CachedPlan;
+/* GUC variables */
+extern int min_cached_plans;
+extern int plancache_prune_min_age;
extern void InitPlanCache(void);
extern void ResetPlanCache(void);
--
2.16.3
----Next_Part(Tue_Jul_03_09_29_14_2018_474)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
filename="v5-0002-introduce-dynhash-pruning.patch"
^ permalink raw reply [nested|flat] 9+ messages in thread
* [PATCH 1/3] Remove entries that haven't been used for a certain time
@ 2017-12-26 08:43 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 9+ messages in thread
From: Kyotaro Horiguchi @ 2017-12-26 08:43 UTC (permalink / raw)
Catcache entries can be left alone for several reasons. It is not
desirable that they eat up memory. With this patch, This adds
consideration of removal of entries that haven't been used for a
certain time before enlarging the hash array.
---
doc/src/sgml/config.sgml | 38 ++++++
src/backend/access/transam/xact.c | 3 +
src/backend/utils/cache/catcache.c | 153 +++++++++++++++++++++++-
src/backend/utils/cache/plancache.c | 163 ++++++++++++++++++++++++++
src/backend/utils/misc/guc.c | 33 ++++++
src/backend/utils/misc/postgresql.conf.sample | 2 +
src/include/utils/catcache.h | 19 +++
src/include/utils/plancache.h | 7 +-
8 files changed, 413 insertions(+), 5 deletions(-)
diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index 7bfbc87109..4ba4327007 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -1617,6 +1617,44 @@ include_dir 'conf.d'
</listitem>
</varlistentry>
+ <varlistentry id="guc-syscache-memory-target" xreflabel="syscache_memory_target">
+ <term><varname>syscache_memory_target</varname> (<type>integer</type>)
+ <indexterm>
+ <primary><varname>syscache_memory_target</varname> configuration parameter</primary>
+ </indexterm>
+ </term>
+ <listitem>
+ <para>
+ Specifies the maximum amount of memory to which syscache is expanded
+ without pruning. The value defaults to 0, indicating that pruning is
+ always considered. After exceeding this size, syscache pruning is
+ considered according to
+ <xref linkend="guc-syscache-prune-min-age"/>. If you need to keep
+ certain amount of syscache entries with intermittent usage, try
+ increase this setting.
+ </para>
+ </listitem>
+ </varlistentry>
+
+ <varlistentry id="guc-syscache-prune-min-age" xreflabel="syscache_prune_min_age">
+ <term><varname>syscache_prune_min_age</varname> (<type>integer</type>)
+ <indexterm>
+ <primary><varname>syscache_prune_min_age</varname> configuration parameter</primary>
+ </indexterm>
+ </term>
+ <listitem>
+ <para>
+ Specifies the minimum amount of unused time in seconds at which a
+ syscache entry is considered to be removed. -1 indicates that syscache
+ pruning is disabled at all. The value defaults to 600 seconds
+ (<literal>10 minutes</literal>). The syscache entries that are not
+ used for the duration can be removed to prevent syscache bloat. This
+ behavior is suppressed until the size of syscache exceeds
+ <xref linkend="guc-syscache-memory-target"/>.
+ </para>
+ </listitem>
+ </varlistentry>
+
<varlistentry id="guc-max-stack-depth" xreflabel="max_stack_depth">
<term><varname>max_stack_depth</varname> (<type>integer</type>)
<indexterm>
diff --git a/src/backend/access/transam/xact.c b/src/backend/access/transam/xact.c
index 8e6aef332c..e4a4a5874c 100644
--- a/src/backend/access/transam/xact.c
+++ b/src/backend/access/transam/xact.c
@@ -732,6 +732,9 @@ void
SetCurrentStatementStartTimestamp(void)
{
stmtStartTimestamp = GetCurrentTimestamp();
+
+ /* Set this timestamp as aproximated current time */
+ SetCatCacheClock(stmtStartTimestamp);
}
/*
diff --git a/src/backend/utils/cache/catcache.c b/src/backend/utils/cache/catcache.c
index 5ddbf6eab1..9f421cd242 100644
--- a/src/backend/utils/cache/catcache.c
+++ b/src/backend/utils/cache/catcache.c
@@ -71,9 +71,24 @@
#define CACHE6_elog(a,b,c,d,e,f,g)
#endif
+/*
+ * GUC variable to define the minimum size of hash to cosider entry eviction.
+ * This variable is shared among various cache mechanisms.
+ */
+int cache_memory_target = 0;
+
+/* GUC variable to define the minimum age of entries that will be cosidered to
+ * be evicted in seconds. This variable is shared among various cache
+ * mechanisms.
+ */
+int cache_prune_min_age = 600;
+
/* Cache management header --- pointer is NULL until created */
static CatCacheHeader *CacheHdr = NULL;
+/* Timestamp used for any operation on caches. */
+TimestampTz catcacheclock = 0;
+
static inline HeapTuple SearchCatCacheInternal(CatCache *cache,
int nkeys,
Datum v1, Datum v2,
@@ -866,9 +881,130 @@ InitCatCache(int id,
*/
MemoryContextSwitchTo(oldcxt);
+ /* initilize catcache reference clock if haven't done yet */
+ if (catcacheclock == 0)
+ catcacheclock = GetCurrentTimestamp();
+
return cp;
}
+/*
+ * CatCacheCleanupOldEntries - Remove infrequently-used entries
+ *
+ * Catcache entries can be left alone for several reasons. We remove them if
+ * they are not accessed for a certain time to prevent catcache from
+ * bloating. The eviction is performed with the similar algorithm with buffer
+ * eviction using access counter. Entries that are accessed several times can
+ * live longer than those that have had no access in the same duration.
+ */
+static bool
+CatCacheCleanupOldEntries(CatCache *cp)
+{
+ int i;
+ int nremoved = 0;
+ size_t hash_size;
+#ifdef CATCACHE_STATS
+ /* These variables are only for debugging purpose */
+ int ntotal = 0;
+ /*
+ * nth element in nentries stores the number of cache entries that have
+ * lived unaccessed for corresponding multiple in ageclass of
+ * cache_prune_min_age. The index of nremoved_entry is the value of the
+ * clock-sweep counter, which takes from 0 up to 2.
+ */
+ double ageclass[] = {0.05, 0.1, 1.0, 2.0, 3.0, 0.0};
+ int nentries[] = {0, 0, 0, 0, 0, 0};
+ int nremoved_entry[3] = {0, 0, 0};
+ int j;
+#endif
+
+ /* Return immediately if no pruning is wanted */
+ if (cache_prune_min_age < 0)
+ return false;
+
+ /*
+ * Return without pruning if the size of the hash is below the target.
+ * Since the area for bucket array is dominant, consider only it.
+ */
+ hash_size = cp->cc_nbuckets * sizeof(dlist_head);
+ if (hash_size < (Size) cache_memory_target * 1024L)
+ return false;
+
+ /* Search the whole hash for entries to remove */
+ for (i = 0; i < cp->cc_nbuckets; i++)
+ {
+ dlist_mutable_iter iter;
+
+ dlist_foreach_modify(iter, &cp->cc_bucket[i])
+ {
+ CatCTup *ct = dlist_container(CatCTup, cache_elem, iter.cur);
+ long entry_age;
+ int us;
+
+
+ /*
+ * Calculate the duration from the time of the last access to the
+ * "current" time. Since catcacheclock is not advanced within a
+ * transaction, the entries that are accessed within the current
+ * transaction won't be pruned.
+ */
+ TimestampDifference(ct->lastaccess, catcacheclock, &entry_age, &us);
+
+#ifdef CATCACHE_STATS
+ /* count catcache entries for each age class */
+ ntotal++;
+ for (j = 0 ;
+ ageclass[j] != 0.0 &&
+ entry_age > cache_prune_min_age * ageclass[j] ;
+ j++);
+ if (ageclass[j] == 0.0) j--;
+ nentries[j]++;
+#endif
+
+ /*
+ * Try to remove entries older than cache_prune_min_age seconds.
+ * Entries that are not accessed after last pruning are removed in
+ * that seconds, and that has been accessed several times are
+ * removed after leaving alone for up to three times of the
+ * duration. We don't try shrink buckets since pruning effectively
+ * caps catcache expansion in the long term.
+ */
+ if (entry_age > cache_prune_min_age)
+ {
+#ifdef CATCACHE_STATS
+ Assert (ct->naccess >= 0 && ct->naccess <= 2);
+ nremoved_entry[ct->naccess]++;
+#endif
+ if (ct->naccess > 0)
+ ct->naccess--;
+ else
+ {
+ if (!ct->c_list || ct->c_list->refcount == 0)
+ {
+ CatCacheRemoveCTup(cp, ct);
+ nremoved++;
+ }
+ }
+ }
+ }
+ }
+
+#ifdef CATCACHE_STATS
+ ereport(DEBUG1,
+ (errmsg ("removed %d/%d, age(-%.0fs:%d, -%.0fs:%d, *-%.0fs:%d, -%.0fs:%d, -%.0fs:%d) naccessed(0:%d, 1:%d, 2:%d)",
+ nremoved, ntotal,
+ ageclass[0] * cache_prune_min_age, nentries[0],
+ ageclass[1] * cache_prune_min_age, nentries[1],
+ ageclass[2] * cache_prune_min_age, nentries[2],
+ ageclass[3] * cache_prune_min_age, nentries[3],
+ ageclass[4] * cache_prune_min_age, nentries[4],
+ nremoved_entry[0], nremoved_entry[1], nremoved_entry[2]),
+ errhidestmt(true)));
+#endif
+
+ return nremoved > 0;
+}
+
/*
* Enlarge a catcache, doubling the number of buckets.
*/
@@ -1282,6 +1418,11 @@ SearchCatCacheInternal(CatCache *cache,
*/
dlist_move_head(bucket, &ct->cache_elem);
+ /* Update access information for pruning */
+ if (ct->naccess < 2)
+ ct->naccess++;
+ ct->lastaccess = catcacheclock;
+
/*
* If it's a positive entry, bump its refcount and return it. If it's
* negative, we can report failure to the caller.
@@ -1813,7 +1954,6 @@ ReleaseCatCacheList(CatCList *list)
CatCacheRemoveCList(list->my_cache, list);
}
-
/*
* CatalogCacheCreateEntry
* Create a new CatCTup entry, copying the given HeapTuple and other
@@ -1906,6 +2046,8 @@ CatalogCacheCreateEntry(CatCache *cache, HeapTuple ntp, Datum *arguments,
ct->dead = false;
ct->negative = negative;
ct->hash_value = hashValue;
+ ct->naccess = 0;
+ ct->lastaccess = catcacheclock;
dlist_push_head(&cache->cc_bucket[hashIndex], &ct->cache_elem);
@@ -1913,10 +2055,13 @@ CatalogCacheCreateEntry(CatCache *cache, HeapTuple ntp, Datum *arguments,
CacheHdr->ch_ntup++;
/*
- * If the hash table has become too full, enlarge the buckets array. Quite
- * arbitrarily, we enlarge when fill factor > 2.
+ * If the hash table has become too full, try cleanup by removing
+ * infrequently used entries to make a room for the new entry. If it
+ * failed, enlarge the bucket array instead. Quite arbitrarily, we try
+ * this when fill factor > 2.
*/
- if (cache->cc_ntup > cache->cc_nbuckets * 2)
+ if (cache->cc_ntup > cache->cc_nbuckets * 2 &&
+ !CatCacheCleanupOldEntries(cache))
RehashCatCache(cache);
return ct;
diff --git a/src/backend/utils/cache/plancache.c b/src/backend/utils/cache/plancache.c
index 0ad3e3c736..701ead152c 100644
--- a/src/backend/utils/cache/plancache.c
+++ b/src/backend/utils/cache/plancache.c
@@ -63,12 +63,14 @@
#include "storage/lmgr.h"
#include "tcop/pquery.h"
#include "tcop/utility.h"
+#include "utils/catcache.h"
#include "utils/inval.h"
#include "utils/memutils.h"
#include "utils/resowner_private.h"
#include "utils/rls.h"
#include "utils/snapmgr.h"
#include "utils/syscache.h"
+#include "utils/timestamp.h"
/*
@@ -86,6 +88,12 @@
* guarantee to save a CachedPlanSource without error.
*/
static CachedPlanSource *first_saved_plan = NULL;
+static CachedPlanSource *last_saved_plan = NULL;
+static int num_saved_plans = 0;
+static TimestampTz oldest_saved_plan = 0;
+
+/* GUC variables */
+int min_cached_plans = 1000;
static void ReleaseGenericPlan(CachedPlanSource *plansource);
static List *RevalidateCachedQuery(CachedPlanSource *plansource,
@@ -105,6 +113,7 @@ static TupleDesc PlanCacheComputeResultDesc(List *stmt_list);
static void PlanCacheRelCallback(Datum arg, Oid relid);
static void PlanCacheFuncCallback(Datum arg, int cacheid, uint32 hashvalue);
static void PlanCacheSysCallback(Datum arg, int cacheid, uint32 hashvalue);
+static void PruneCachedPlan(void);
/*
@@ -208,6 +217,8 @@ CreateCachedPlan(RawStmt *raw_parse_tree,
plansource->generic_cost = -1;
plansource->total_custom_cost = 0;
plansource->num_custom_plans = 0;
+ plansource->last_access = GetCatCacheClock();
+
MemoryContextSwitchTo(oldcxt);
@@ -423,6 +434,28 @@ CompleteCachedPlan(CachedPlanSource *plansource,
plansource->is_valid = true;
}
+/* moves the plansource to the first in the list */
+static inline void
+MovePlansourceToFirst(CachedPlanSource *plansource)
+{
+ if (first_saved_plan != plansource)
+ {
+ /* delink this element */
+ if (plansource->next_saved)
+ plansource->next_saved->prev_saved = plansource->prev_saved;
+ if (plansource->prev_saved)
+ plansource->prev_saved->next_saved = plansource->next_saved;
+ if (last_saved_plan == plansource)
+ last_saved_plan = plansource->prev_saved;
+
+ /* insert at the beginning */
+ first_saved_plan->prev_saved = plansource;
+ plansource->next_saved = first_saved_plan;
+ plansource->prev_saved = NULL;
+ first_saved_plan = plansource;
+ }
+}
+
/*
* SaveCachedPlan: save a cached plan permanently
*
@@ -470,6 +503,11 @@ SaveCachedPlan(CachedPlanSource *plansource)
* Add the entry to the global list of cached plans.
*/
plansource->next_saved = first_saved_plan;
+ if (first_saved_plan)
+ first_saved_plan->prev_saved = plansource;
+ else
+ last_saved_plan = plansource;
+ plansource->prev_saved = NULL;
first_saved_plan = plansource;
plansource->is_saved = true;
@@ -492,7 +530,11 @@ DropCachedPlan(CachedPlanSource *plansource)
if (plansource->is_saved)
{
if (first_saved_plan == plansource)
+ {
first_saved_plan = plansource->next_saved;
+ if (first_saved_plan)
+ first_saved_plan->prev_saved = NULL;
+ }
else
{
CachedPlanSource *psrc;
@@ -502,10 +544,19 @@ DropCachedPlan(CachedPlanSource *plansource)
if (psrc->next_saved == plansource)
{
psrc->next_saved = plansource->next_saved;
+ if (psrc->next_saved)
+ psrc->next_saved->prev_saved = psrc;
break;
}
}
}
+
+ if (last_saved_plan == plansource)
+ {
+ last_saved_plan = plansource->prev_saved;
+ if (last_saved_plan)
+ last_saved_plan->next_saved = NULL;
+ }
plansource->is_saved = false;
}
@@ -537,6 +588,13 @@ ReleaseGenericPlan(CachedPlanSource *plansource)
Assert(plan->magic == CACHEDPLAN_MAGIC);
plansource->gplan = NULL;
ReleaseCachedPlan(plan, false);
+
+ /* decrement "saved plans" counter */
+ if (plansource->is_saved)
+ {
+ Assert (num_saved_plans > 0);
+ num_saved_plans--;
+ }
}
}
@@ -1148,6 +1206,17 @@ GetCachedPlan(CachedPlanSource *plansource, ParamListInfo boundParams,
if (useResOwner && !plansource->is_saved)
elog(ERROR, "cannot apply ResourceOwner to non-saved cached plan");
+ /*
+ * set last-accessed timestamp and move this plan to the first of the list
+ */
+ if (plansource->is_saved)
+ {
+ plansource->last_access = GetCatCacheClock();
+
+ /* move this plan to the first of the list */
+ MovePlansourceToFirst(plansource);
+ }
+
/* Make sure the querytree list is valid and we have parse-time locks */
qlist = RevalidateCachedQuery(plansource, queryEnv);
@@ -1156,6 +1225,11 @@ GetCachedPlan(CachedPlanSource *plansource, ParamListInfo boundParams,
if (!customplan)
{
+ /* Prune cached plans if needed */
+ if (plansource->is_saved &&
+ min_cached_plans >= 0 && num_saved_plans > min_cached_plans)
+ PruneCachedPlan();
+
if (CheckCachedPlan(plansource))
{
/* We want a generic plan, and we already have a valid one */
@@ -1168,6 +1242,11 @@ GetCachedPlan(CachedPlanSource *plansource, ParamListInfo boundParams,
plan = BuildCachedPlan(plansource, qlist, NULL, queryEnv);
/* Just make real sure plansource->gplan is clear */
ReleaseGenericPlan(plansource);
+
+ /* count this new saved plan */
+ if (plansource->is_saved)
+ num_saved_plans++;
+
/* Link the new generic plan into the plansource */
plansource->gplan = plan;
plan->refcount++;
@@ -1856,6 +1935,90 @@ PlanCacheSysCallback(Datum arg, int cacheid, uint32 hashvalue)
ResetPlanCache();
}
+/*
+ * PrunePlanCache: removes generic plan of "old" saved plans.
+ */
+static void
+PruneCachedPlan(void)
+{
+ CachedPlanSource *plansource;
+ TimestampTz currclock = GetCatCacheClock();
+ long age;
+ int us;
+ int nremoved = 0;
+
+ /* do nothing if not wanted */
+ if (cache_prune_min_age < 0 || num_saved_plans <= min_cached_plans)
+ return;
+
+ /* Fast check for oldest cache */
+ if (oldest_saved_plan > 0)
+ {
+ TimestampDifference(oldest_saved_plan, currclock, &age, &us);
+ if (age < cache_prune_min_age)
+ return;
+ }
+
+ /* last plan is the oldest. */
+ for (plansource = last_saved_plan; plansource; plansource = plansource->prev_saved)
+ {
+ long plan_age;
+ int us;
+
+ Assert(plansource->magic == CACHEDPLANSOURCE_MAGIC);
+
+ /* we want to prune no more plans */
+ if (num_saved_plans <= min_cached_plans)
+ break;
+
+ /*
+ * No work if it already doesn't have gplan and move it to the
+ * beginning so that we don't see it at the next time
+ */
+ if (!plansource->gplan)
+ continue;
+
+ /*
+ * Check age for pruning. Can exit immediately when finding a
+ * not-older element.
+ */
+ TimestampDifference(plansource->last_access, currclock, &plan_age, &us);
+ if (plan_age <= cache_prune_min_age)
+ {
+ /* this entry is the next oldest */
+ oldest_saved_plan = plansource->last_access;
+ break;
+ }
+
+ /*
+ * Here, remove generic plans of this plansrouceif it is not actually
+ * used and move it to the beginning of the list. Just update
+ * last_access and move it to the beginning if the plan is used.
+ */
+ if (plansource->gplan->refcount <= 1)
+ {
+ ReleaseGenericPlan(plansource);
+ nremoved++;
+ }
+
+ plansource->last_access = currclock;
+ }
+
+ /* move the "removed" plansrouces altogehter to the beginning of the list */
+ if (plansource != last_saved_plan && plansource)
+ {
+ plansource->next_saved->prev_saved = NULL;
+ first_saved_plan->prev_saved = last_saved_plan;
+ last_saved_plan->next_saved = first_saved_plan;
+ first_saved_plan = plansource->next_saved;
+ plansource->next_saved = NULL;
+ last_saved_plan = plansource;
+ }
+
+ if (nremoved > 0)
+ elog(DEBUG1, "plancache removed %d/%d", nremoved, num_saved_plans);
+}
+
/*
* ResetPlanCache: invalidate all cached plans.
*/
diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c
index 859ef931e7..774a87ed2c 100644
--- a/src/backend/utils/misc/guc.c
+++ b/src/backend/utils/misc/guc.c
@@ -79,6 +79,7 @@
#include "tsearch/ts_cache.h"
#include "utils/builtins.h"
#include "utils/bytea.h"
+#include "utils/catcache.h"
#include "utils/guc_tables.h"
#include "utils/memutils.h"
#include "utils/pg_locale.h"
@@ -2105,6 +2106,38 @@ static struct config_int ConfigureNamesInt[] =
NULL, NULL, NULL
},
+ {
+ {"cache_memory_target", PGC_USERSET, RESOURCES_MEM,
+ gettext_noop("Sets the minimum syscache size to keep."),
+ gettext_noop("Cache is not pruned before exceeding this size."),
+ GUC_UNIT_KB
+ },
+ &cache_memory_target,
+ 0, 0, MAX_KILOBYTES,
+ NULL, NULL, NULL
+ },
+
+ {
+ {"cache_prune_min_age", PGC_USERSET, RESOURCES_MEM,
+ gettext_noop("Sets the minimum unused duration of cache entries before removal."),
+ gettext_noop("Cache entries that live unused for longer than this seconds are considered to be removed."),
+ GUC_UNIT_S
+ },
+ &cache_prune_min_age,
+ 600, -1, INT_MAX,
+ NULL, NULL, NULL
+ },
+
+ {
+ {"min_cached_plans", PGC_USERSET, RESOURCES_MEM,
+ gettext_noop("Sets the minimum number of cached plans kept on memory."),
+ gettext_noop("Timeout invalidation of plancache is not activated until the number of plancaches reaches this value. -1 means timeout invalidation is always active.")
+ },
+ &min_cached_plans,
+ 1000, -1, INT_MAX,
+ NULL, NULL, NULL
+ },
+
/*
* We use the hopefully-safely-small value of 100kB as the compiled-in
* default for max_stack_depth. InitializeGUCOptions will increase it if
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index 9e39baf466..3f2760ef9d 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -126,6 +126,8 @@
#work_mem = 4MB # min 64kB
#maintenance_work_mem = 64MB # min 1MB
#autovacuum_work_mem = -1 # min 1MB, or -1 to use maintenance_work_mem
+#cache_memory_target = 0kB # in kB
+#cache_prune_min_age = 600s # -1 disables pruning
#max_stack_depth = 2MB # min 100kB
#dynamic_shared_memory_type = posix # the default is the first option
# supported by the operating system:
diff --git a/src/include/utils/catcache.h b/src/include/utils/catcache.h
index 7b22f9c7bc..599303be56 100644
--- a/src/include/utils/catcache.h
+++ b/src/include/utils/catcache.h
@@ -22,6 +22,7 @@
#include "access/htup.h"
#include "access/skey.h"
+#include "datatype/timestamp.h"
#include "lib/ilist.h"
#include "utils/relcache.h"
@@ -119,6 +120,8 @@ typedef struct catctup
bool dead; /* dead but not yet removed? */
bool negative; /* negative cache entry? */
HeapTupleData tuple; /* tuple management header */
+ int naccess; /* # of access to this entry, up to 2 */
+ TimestampTz lastaccess; /* approx. timestamp of the last usage */
/*
* The tuple may also be a member of at most one CatCList. (If a single
@@ -189,6 +192,22 @@ typedef struct catcacheheader
/* this extern duplicates utils/memutils.h... */
extern PGDLLIMPORT MemoryContext CacheMemoryContext;
+/* for guc.c, not PGDLLPMPORT'ed */
+extern int cache_prune_min_age;
+extern int cache_memory_target;
+
+/* to use as access timestamp of catcache entries */
+extern TimestampTz catcacheclock;
+
+/*
+ * SetCatCacheClock - set timestamp for catcache access record
+ */
+static inline void
+SetCatCacheClock(TimestampTz ts)
+{
+ catcacheclock = ts;
+}
+
extern void CreateCacheMemoryContext(void);
extern CatCache *InitCatCache(int id, Oid reloid, Oid indexoid,
diff --git a/src/include/utils/plancache.h b/src/include/utils/plancache.h
index ab20aa04b0..f3c5b2010d 100644
--- a/src/include/utils/plancache.h
+++ b/src/include/utils/plancache.h
@@ -110,11 +110,13 @@ typedef struct CachedPlanSource
bool is_valid; /* is the query_list currently valid? */
int generation; /* increments each time we create a plan */
/* If CachedPlanSource has been saved, it is a member of a global list */
- struct CachedPlanSource *next_saved; /* list link, if so */
+ struct CachedPlanSource *prev_saved; /* list prev link, if so */
+ struct CachedPlanSource *next_saved; /* list next link, if so */
/* State kept to help decide whether to use custom or generic plans: */
double generic_cost; /* cost of generic plan, or -1 if not known */
double total_custom_cost; /* total cost of custom plans so far */
int num_custom_plans; /* number of plans included in total */
+ TimestampTz last_access; /* timestamp of the last usage */
} CachedPlanSource;
/*
@@ -143,6 +145,9 @@ typedef struct CachedPlan
MemoryContext context; /* context containing this CachedPlan */
} CachedPlan;
+/* GUC variables */
+extern int min_cached_plans;
+extern int plancache_prune_min_age;
extern void InitPlanCache(void);
extern void ResetPlanCache(void);
--
2.16.3
----Next_Part(Tue_Jun_26_18_00_03_2018_442)----
^ permalink raw reply [nested|flat] 9+ messages in thread
* [PATCH 1/3] Remove entries that haven't been used for a certain time
@ 2018-10-16 04:04 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 9+ messages in thread
From: Kyotaro Horiguchi @ 2018-10-16 04:04 UTC (permalink / raw)
Catcache entries can be left alone for several reasons. It is not
desirable that they eat up memory. With this patch, This adds
consideration of removal of entries that haven't been used for a
certain time before enlarging the hash array.
---
doc/src/sgml/config.sgml | 38 ++++++
src/backend/access/transam/xact.c | 5 +
src/backend/utils/cache/catcache.c | 166 ++++++++++++++++++++++++--
src/backend/utils/misc/guc.c | 23 ++++
src/backend/utils/misc/postgresql.conf.sample | 2 +
src/include/utils/catcache.h | 28 ++++-
6 files changed, 254 insertions(+), 8 deletions(-)
diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index 7554cba3f9..c3133d742b 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -1618,6 +1618,44 @@ include_dir 'conf.d'
</listitem>
</varlistentry>
+ <varlistentry id="guc-syscache-memory-target" xreflabel="syscache_memory_target">
+ <term><varname>syscache_memory_target</varname> (<type>integer</type>)
+ <indexterm>
+ <primary><varname>syscache_memory_target</varname> configuration parameter</primary>
+ </indexterm>
+ </term>
+ <listitem>
+ <para>
+ Specifies the maximum amount of memory to which syscache is expanded
+ without pruning. The value defaults to 0, indicating that pruning is
+ always considered. After exceeding this size, syscache pruning is
+ considered according to
+ <xref linkend="guc-syscache-prune-min-age"/>. If you need to keep
+ certain amount of syscache entries with intermittent usage, try
+ increase this setting.
+ </para>
+ </listitem>
+ </varlistentry>
+
+ <varlistentry id="guc-syscache-prune-min-age" xreflabel="syscache_prune_min_age">
+ <term><varname>syscache_prune_min_age</varname> (<type>integer</type>)
+ <indexterm>
+ <primary><varname>syscache_prune_min_age</varname> configuration parameter</primary>
+ </indexterm>
+ </term>
+ <listitem>
+ <para>
+ Specifies the minimum amount of unused time in seconds at which a
+ syscache entry is considered to be removed. -1 indicates that syscache
+ pruning is disabled at all. The value defaults to 600 seconds
+ (<literal>10 minutes</literal>). The syscache entries that are not
+ used for the duration can be removed to prevent syscache bloat. This
+ behavior is suppressed until the size of syscache exceeds
+ <xref linkend="guc-syscache-memory-target"/>.
+ </para>
+ </listitem>
+ </varlistentry>
+
<varlistentry id="guc-max-stack-depth" xreflabel="max_stack_depth">
<term><varname>max_stack_depth</varname> (<type>integer</type>)
<indexterm>
diff --git a/src/backend/access/transam/xact.c b/src/backend/access/transam/xact.c
index 8c1621d949..083b6dc7aa 100644
--- a/src/backend/access/transam/xact.c
+++ b/src/backend/access/transam/xact.c
@@ -733,7 +733,12 @@ void
SetCurrentStatementStartTimestamp(void)
{
if (!IsParallelWorker())
+ {
stmtStartTimestamp = GetCurrentTimestamp();
+
+ /* Set this timestamp as aproximated current time */
+ SetCatCacheClock(stmtStartTimestamp);
+ }
else
Assert(stmtStartTimestamp != 0);
}
diff --git a/src/backend/utils/cache/catcache.c b/src/backend/utils/cache/catcache.c
index 5ddbf6eab1..9be463311d 100644
--- a/src/backend/utils/cache/catcache.c
+++ b/src/backend/utils/cache/catcache.c
@@ -71,9 +71,24 @@
#define CACHE6_elog(a,b,c,d,e,f,g)
#endif
+/*
+ * GUC variable to define the minimum size of hash to cosider entry eviction.
+ * This variable is shared among various cache mechanisms.
+ */
+int cache_memory_target = 0;
+
+/* GUC variable to define the minimum age of entries that will be cosidered to
+ * be evicted in seconds. This variable is shared among various cache
+ * mechanisms.
+ */
+int cache_prune_min_age = 600;
+
/* Cache management header --- pointer is NULL until created */
static CatCacheHeader *CacheHdr = NULL;
+/* Timestamp used for any operation on caches. */
+TimestampTz catcacheclock = 0;
+
static inline HeapTuple SearchCatCacheInternal(CatCache *cache,
int nkeys,
Datum v1, Datum v2,
@@ -498,6 +513,7 @@ CatCacheRemoveCTup(CatCache *cache, CatCTup *ct)
CatCacheFreeKeys(cache->cc_tupdesc, cache->cc_nkeys,
cache->cc_keyno, ct->keys);
+ cache->cc_tupsize -= ct->size;
pfree(ct);
--cache->cc_ntup;
@@ -849,6 +865,7 @@ InitCatCache(int id,
cp->cc_nkeys = nkeys;
for (i = 0; i < nkeys; ++i)
cp->cc_keyno[i] = key[i];
+ cp->cc_tupsize = 0;
/*
* new cache is initialized as far as we can go for now. print some
@@ -866,9 +883,129 @@ InitCatCache(int id,
*/
MemoryContextSwitchTo(oldcxt);
+ /* initilize catcache reference clock if haven't done yet */
+ if (catcacheclock == 0)
+ catcacheclock = GetCurrentTimestamp();
+
return cp;
}
+/*
+ * CatCacheCleanupOldEntries - Remove infrequently-used entries
+ *
+ * Catcache entries can be left alone for several reasons. We remove them if
+ * they are not accessed for a certain time to prevent catcache from
+ * bloating. The eviction is performed with the similar algorithm with buffer
+ * eviction using access counter. Entries that are accessed several times can
+ * live longer than those that have had no access in the same duration.
+ */
+static bool
+CatCacheCleanupOldEntries(CatCache *cp)
+{
+ int i;
+ int nremoved = 0;
+ size_t hash_size;
+#ifdef CATCACHE_STATS
+ /* These variables are only for debugging purpose */
+ int ntotal = 0;
+ /*
+ * nth element in nentries stores the number of cache entries that have
+ * lived unaccessed for corresponding multiple in ageclass of
+ * cache_prune_min_age. The index of nremoved_entry is the value of the
+ * clock-sweep counter, which takes from 0 up to 2.
+ */
+ double ageclass[] = {0.05, 0.1, 1.0, 2.0, 3.0, 0.0};
+ int nentries[] = {0, 0, 0, 0, 0, 0};
+ int nremoved_entry[3] = {0, 0, 0};
+ int j;
+#endif
+
+ /* Return immediately if no pruning is wanted */
+ if (cache_prune_min_age < 0)
+ return false;
+
+ /*
+ * Return without pruning if the size of the hash is below the target.
+ */
+ hash_size = cp->cc_nbuckets * sizeof(dlist_head);
+ if (hash_size + cp->cc_tupsize < (Size) cache_memory_target * 1024L)
+ return false;
+
+ /* Search the whole hash for entries to remove */
+ for (i = 0; i < cp->cc_nbuckets; i++)
+ {
+ dlist_mutable_iter iter;
+
+ dlist_foreach_modify(iter, &cp->cc_bucket[i])
+ {
+ CatCTup *ct = dlist_container(CatCTup, cache_elem, iter.cur);
+ long entry_age;
+ int us;
+
+
+ /*
+ * Calculate the duration from the time of the last access to the
+ * "current" time. Since catcacheclock is not advanced within a
+ * transaction, the entries that are accessed within the current
+ * transaction won't be pruned.
+ */
+ TimestampDifference(ct->lastaccess, catcacheclock, &entry_age, &us);
+
+#ifdef CATCACHE_STATS
+ /* count catcache entries for each age class */
+ ntotal++;
+ for (j = 0 ;
+ ageclass[j] != 0.0 &&
+ entry_age > cache_prune_min_age * ageclass[j] ;
+ j++);
+ if (ageclass[j] == 0.0) j--;
+ nentries[j]++;
+#endif
+
+ /*
+ * Try to remove entries older than cache_prune_min_age seconds.
+ * Entries that are not accessed after last pruning are removed in
+ * that seconds, and that has been accessed several times are
+ * removed after leaving alone for up to three times of the
+ * duration. We don't try shrink buckets since pruning effectively
+ * caps catcache expansion in the long term.
+ */
+ if (entry_age > cache_prune_min_age)
+ {
+#ifdef CATCACHE_STATS
+ Assert (ct->naccess >= 0 && ct->naccess <= 2);
+ nremoved_entry[ct->naccess]++;
+#endif
+ if (ct->naccess > 0)
+ ct->naccess--;
+ else
+ {
+ if (!ct->c_list || ct->c_list->refcount == 0)
+ {
+ CatCacheRemoveCTup(cp, ct);
+ nremoved++;
+ }
+ }
+ }
+ }
+ }
+
+#ifdef CATCACHE_STATS
+ ereport(DEBUG1,
+ (errmsg ("removed %d/%d, age(-%.0fs:%d, -%.0fs:%d, *-%.0fs:%d, -%.0fs:%d, -%.0fs:%d) naccessed(0:%d, 1:%d, 2:%d)",
+ nremoved, ntotal,
+ ageclass[0] * cache_prune_min_age, nentries[0],
+ ageclass[1] * cache_prune_min_age, nentries[1],
+ ageclass[2] * cache_prune_min_age, nentries[2],
+ ageclass[3] * cache_prune_min_age, nentries[3],
+ ageclass[4] * cache_prune_min_age, nentries[4],
+ nremoved_entry[0], nremoved_entry[1], nremoved_entry[2]),
+ errhidestmt(true)));
+#endif
+
+ return nremoved > 0;
+}
+
/*
* Enlarge a catcache, doubling the number of buckets.
*/
@@ -1282,6 +1419,11 @@ SearchCatCacheInternal(CatCache *cache,
*/
dlist_move_head(bucket, &ct->cache_elem);
+ /* Update access information for pruning */
+ if (ct->naccess < 2)
+ ct->naccess++;
+ ct->lastaccess = catcacheclock;
+
/*
* If it's a positive entry, bump its refcount and return it. If it's
* negative, we can report failure to the caller.
@@ -1827,11 +1969,13 @@ CatalogCacheCreateEntry(CatCache *cache, HeapTuple ntp, Datum *arguments,
CatCTup *ct;
HeapTuple dtp;
MemoryContext oldcxt;
+ int tupsize = 0;
/* negative entries have no tuple associated */
if (ntp)
{
int i;
+ int tupsize;
Assert(!negative);
@@ -1850,13 +1994,14 @@ CatalogCacheCreateEntry(CatCache *cache, HeapTuple ntp, Datum *arguments,
/* Allocate memory for CatCTup and the cached tuple in one go */
oldcxt = MemoryContextSwitchTo(CacheMemoryContext);
- ct = (CatCTup *) palloc(sizeof(CatCTup) +
- MAXIMUM_ALIGNOF + dtp->t_len);
+ tupsize = sizeof(CatCTup) + MAXIMUM_ALIGNOF + dtp->t_len;
+ ct = (CatCTup *) palloc(tupsize);
ct->tuple.t_len = dtp->t_len;
ct->tuple.t_self = dtp->t_self;
ct->tuple.t_tableOid = dtp->t_tableOid;
ct->tuple.t_data = (HeapTupleHeader)
MAXALIGN(((char *) ct) + sizeof(CatCTup));
+ ct->size = tupsize;
/* copy tuple contents */
memcpy((char *) ct->tuple.t_data,
(const char *) dtp->t_data,
@@ -1884,8 +2029,8 @@ CatalogCacheCreateEntry(CatCache *cache, HeapTuple ntp, Datum *arguments,
{
Assert(negative);
oldcxt = MemoryContextSwitchTo(CacheMemoryContext);
- ct = (CatCTup *) palloc(sizeof(CatCTup));
-
+ tupsize = sizeof(CatCTup);
+ ct = (CatCTup *) palloc(tupsize);
/*
* Store keys - they'll point into separately allocated memory if not
* by-value.
@@ -1906,17 +2051,24 @@ CatalogCacheCreateEntry(CatCache *cache, HeapTuple ntp, Datum *arguments,
ct->dead = false;
ct->negative = negative;
ct->hash_value = hashValue;
+ ct->naccess = 0;
+ ct->lastaccess = catcacheclock;
+ ct->size = tupsize;
dlist_push_head(&cache->cc_bucket[hashIndex], &ct->cache_elem);
cache->cc_ntup++;
CacheHdr->ch_ntup++;
+ cache->cc_tupsize += tupsize;
/*
- * If the hash table has become too full, enlarge the buckets array. Quite
- * arbitrarily, we enlarge when fill factor > 2.
+ * If the hash table has become too full, try cleanup by removing
+ * infrequently used entries to make a room for the new entry. If it
+ * failed, enlarge the bucket array instead. Quite arbitrarily, we try
+ * this when fill factor > 2.
*/
- if (cache->cc_ntup > cache->cc_nbuckets * 2)
+ if (cache->cc_ntup > cache->cc_nbuckets * 2 &&
+ !CatCacheCleanupOldEntries(cache))
RehashCatCache(cache);
return ct;
diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c
index 2317e8be6b..1a49d576fa 100644
--- a/src/backend/utils/misc/guc.c
+++ b/src/backend/utils/misc/guc.c
@@ -80,6 +80,7 @@
#include "tsearch/ts_cache.h"
#include "utils/builtins.h"
#include "utils/bytea.h"
+#include "utils/catcache.h"
#include "utils/guc_tables.h"
#include "utils/float.h"
#include "utils/memutils.h"
@@ -2113,6 +2114,28 @@ static struct config_int ConfigureNamesInt[] =
NULL, NULL, NULL
},
+ {
+ {"cache_memory_target", PGC_USERSET, RESOURCES_MEM,
+ gettext_noop("Sets the minimum syscache size to keep."),
+ gettext_noop("Cache is not pruned before exceeding this size."),
+ GUC_UNIT_KB
+ },
+ &cache_memory_target,
+ 0, 0, MAX_KILOBYTES,
+ NULL, NULL, NULL
+ },
+
+ {
+ {"cache_prune_min_age", PGC_USERSET, RESOURCES_MEM,
+ gettext_noop("Sets the minimum unused duration of cache entries before removal."),
+ gettext_noop("Cache entries that live unused for longer than this seconds are considered to be removed."),
+ GUC_UNIT_S
+ },
+ &cache_prune_min_age,
+ 600, -1, INT_MAX,
+ NULL, NULL, NULL
+ },
+
/*
* We use the hopefully-safely-small value of 100kB as the compiled-in
* default for max_stack_depth. InitializeGUCOptions will increase it if
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index 4e61bc6521..c59dd898ac 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -126,6 +126,8 @@
#work_mem = 4MB # min 64kB
#maintenance_work_mem = 64MB # min 1MB
#autovacuum_work_mem = -1 # min 1MB, or -1 to use maintenance_work_mem
+#cache_memory_target = 0kB # in kB
+#cache_prune_min_age = 600s # -1 disables pruning
#max_stack_depth = 2MB # min 100kB
#dynamic_shared_memory_type = posix # the default is the first option
# supported by the operating system:
diff --git a/src/include/utils/catcache.h b/src/include/utils/catcache.h
index 7b22f9c7bc..ace4178619 100644
--- a/src/include/utils/catcache.h
+++ b/src/include/utils/catcache.h
@@ -22,6 +22,7 @@
#include "access/htup.h"
#include "access/skey.h"
+#include "datatype/timestamp.h"
#include "lib/ilist.h"
#include "utils/relcache.h"
@@ -61,6 +62,7 @@ typedef struct catcache
slist_node cc_next; /* list link */
ScanKeyData cc_skey[CATCACHE_MAXKEYS]; /* precomputed key info for heap
* scans */
+ int cc_tupsize; /* total amount of catcache tuples */
/*
* Keep these at the end, so that compiling catcache.c with CATCACHE_STATS
@@ -119,7 +121,9 @@ typedef struct catctup
bool dead; /* dead but not yet removed? */
bool negative; /* negative cache entry? */
HeapTupleData tuple; /* tuple management header */
-
+ int naccess; /* # of access to this entry, up to 2 */
+ TimestampTz lastaccess; /* approx. timestamp of the last usage */
+ int size; /* palloc'ed size off this tuple */
/*
* The tuple may also be a member of at most one CatCList. (If a single
* catcache is list-searched with varying numbers of keys, we may have to
@@ -189,6 +193,28 @@ typedef struct catcacheheader
/* this extern duplicates utils/memutils.h... */
extern PGDLLIMPORT MemoryContext CacheMemoryContext;
+/* for guc.c, not PGDLLPMPORT'ed */
+extern int cache_prune_min_age;
+extern int cache_memory_target;
+
+/* to use as access timestamp of catcache entries */
+extern TimestampTz catcacheclock;
+
+/*
+ * SetCatCacheClock - set timestamp for catcache access record
+ */
+static inline void
+SetCatCacheClock(TimestampTz ts)
+{
+ catcacheclock = ts;
+}
+
+static inline TimestampTz
+GetCatCacheClock(void)
+{
+ return catcacheclock;
+}
+
extern void CreateCacheMemoryContext(void);
extern CatCache *InitCatCache(int id, Oid reloid, Oid indexoid,
--
2.16.3
----Next_Part(Fri_Oct_19_17_34_57_2018_852)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
filename="0002-Syscache-usage-tracking-feature.patch"
^ permalink raw reply [nested|flat] 9+ messages in thread
* Re: Readd use of TAP subtests
@ 2022-02-25 14:43 Andrew Dunstan <[email protected]>
2022-02-25 16:35 ` Re: Readd use of TAP subtests Jacob Champion <[email protected]>
2022-02-25 16:41 ` Re: Readd use of TAP subtests Andres Freund <[email protected]>
0 siblings, 2 replies; 9+ messages in thread
From: Andrew Dunstan @ 2022-02-25 14:43 UTC (permalink / raw)
To: Peter Eisentraut <[email protected]>; Andres Freund <[email protected]>; +Cc: pgsql-hackers; Daniel Gustafsson <[email protected]>; Tom Lane <[email protected]>; Dagfinn Ilmari Mannsåker <[email protected]>
On 2/25/22 08:39, Peter Eisentraut wrote:
> On 24.02.22 16:00, Andres Freund wrote:
>> I've incidentally played with subtests yesterdays, when porting
>> src/interfaces/libpq/test/regress.pl to a tap test. Unfortunately it
>> seems
>> that subtests aren't actually specified in the tap format, and that
>> different
>> libraries generate different output formats. The reason this matters
>> somewhat
>> is that meson's testrunner can parse tap and give nicer progress / error
>> reports. But since subtests aren't in the spec it can't currently parse
>> them...
>
> Ok that's good to know. What exactly happens when it tries to parse
> them? Does it not count them or does it fail somehow? The way the
> output is structured
>
> t/001_basic.pl ..
> # Subtest: vacuumlo --help
> ok 1 - exit code 0
> ok 2 - goes to stdout
> ok 3 - nothing to stderr
> 1..3
> ok 1 - vacuumlo --help
>
> it appears that it should be able to parse it nonetheless and should
> just count the non-indented lines.
AIUI TAP consumers are supposed to ignore lines they don't understand.
The Node TAP setup produces output like this, so perl is hardly alone
here. See <https://node-tap.org/docs/api/subtests/;
cheers
andrew
--
Andrew Dunstan
EDB: https://www.enterprisedb.com
^ permalink raw reply [nested|flat] 9+ messages in thread
* Re: Readd use of TAP subtests
2022-02-25 14:43 Re: Readd use of TAP subtests Andrew Dunstan <[email protected]>
@ 2022-02-25 16:35 ` Jacob Champion <[email protected]>
2022-02-25 16:38 ` Re: Readd use of TAP subtests Jacob Champion <[email protected]>
1 sibling, 1 reply; 9+ messages in thread
From: Jacob Champion @ 2022-02-25 16:35 UTC (permalink / raw)
To: [email protected] <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>; +Cc: [email protected] <[email protected]>; [email protected] <[email protected]>; pgsql-hackers; [email protected] <[email protected]>
On Fri, 2022-02-25 at 09:43 -0500, Andrew Dunstan wrote:
> AIUI TAP consumers are supposed to ignore lines they don't understand.
It's undefined behavior [1]:
> Any output that is not a version, a plan, a test line, a YAML block,
> a diagnostic or a bail out is incorrect. How a harness handles the
> incorrect line is undefined. Test::Harness silently ignores incorrect
> lines, but will become more stringent in the future. TAP::Harness
> reports TAP syntax errors at the end of a test run.
--Jacob
[1] https://testanything.org/tap-version-13-specification.html
^ permalink raw reply [nested|flat] 9+ messages in thread
* Re: Readd use of TAP subtests
2022-02-25 14:43 Re: Readd use of TAP subtests Andrew Dunstan <[email protected]>
2022-02-25 16:35 ` Re: Readd use of TAP subtests Jacob Champion <[email protected]>
@ 2022-02-25 16:38 ` Jacob Champion <[email protected]>
0 siblings, 0 replies; 9+ messages in thread
From: Jacob Champion @ 2022-02-25 16:38 UTC (permalink / raw)
To: [email protected] <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>; +Cc: [email protected] <[email protected]>; [email protected] <[email protected]>; pgsql-hackers; [email protected] <[email protected]>
On Fri, 2022-02-25 at 16:35 +0000, Jacob Champion wrote:
> On Fri, 2022-02-25 at 09:43 -0500, Andrew Dunstan wrote:
> > AIUI TAP consumers are supposed to ignore lines they don't understand.
>
> It's undefined behavior [1]:
And of course the minute I send this I notice that I've linked the v13
spec instead of the original... sorry. Assuming Perl isn't marking its
tests as version 13, you are correct:
> Any output line that is not a version, a plan, a test line, a
> diagnostic or a bail out is considered an “unknown” line. A TAP
> parser is required to not consider an unknown line as an error but
> may optionally choose to capture said line and hand it to the test
> harness, which may have custom behavior attached. This is to allow
> for forward compatability. Test::Harness silently ignores incorrect
> lines, but will become more stringent in the future. TAP::Harness
> reports TAP syntax errors at the end of a test run.
--Jacob
^ permalink raw reply [nested|flat] 9+ messages in thread
* Re: Readd use of TAP subtests
2022-02-25 14:43 Re: Readd use of TAP subtests Andrew Dunstan <[email protected]>
@ 2022-02-25 16:41 ` Andres Freund <[email protected]>
2022-02-25 17:28 ` Re: Readd use of TAP subtests Andrew Dunstan <[email protected]>
1 sibling, 1 reply; 9+ messages in thread
From: Andres Freund @ 2022-02-25 16:41 UTC (permalink / raw)
To: Andrew Dunstan <[email protected]>; +Cc: Peter Eisentraut <[email protected]>; pgsql-hackers; Daniel Gustafsson <[email protected]>; Tom Lane <[email protected]>; Dagfinn Ilmari Mannsåker <[email protected]>
Hi,
On 2022-02-25 09:43:20 -0500, Andrew Dunstan wrote:
> AIUI TAP consumers are supposed to ignore lines they don't understand.
Are they?
In http://testanything.org/tap-version-13-specification.html there's:
"Lines written to standard output matching /^(not )?ok\b/ must be interpreted
as test lines. [...]All other lines must not be considered test output."
That says that all other lines aren't "test ouput". But what does that mean?
It certainly doesn't mean they can just be ignored, because obviously
^(TAP version|#|1..|Bail out) isn't to be ignored.
And then there's:
"
Anything else
Any output that is not a version, a plan, a test line, a YAML block, a
diagnostic or a bail out is incorrect. How a harness handles the incorrect
line is undefined. Test::Harness silently ignores incorrect lines, but will
become more stringent in the future. TAP::Harness reports TAP syntax errors at
the end of a test run.
"
If I were to to implement a tap parser this wouldn't make me ignore lines.
Contrasting to that:
http://testanything.org/tap-specification.html
"
Anything else
A TAP parser is required to not consider an unknown line as an error but may
optionally choose to capture said line and hand it to the test harness,
which may have custom behavior attached. This is to allow for forward
compatability. Test::Harness silently ignores incorrect lines, but will
become more stringent in the future. TAP::Harness reports TAP syntax errors
at the end of a test run.
"
I honestly don't know what to make of that. Parsers are supposed to ignore
unknown lines, Test::Harness silently ignores, but also "TAP::Harness reports
TAP syntax errors"? This seems like a self contradictory mess.
Greetings,
Andres Freund
^ permalink raw reply [nested|flat] 9+ messages in thread
* Re: Readd use of TAP subtests
2022-02-25 14:43 Re: Readd use of TAP subtests Andrew Dunstan <[email protected]>
2022-02-25 16:41 ` Re: Readd use of TAP subtests Andres Freund <[email protected]>
@ 2022-02-25 17:28 ` Andrew Dunstan <[email protected]>
0 siblings, 0 replies; 9+ messages in thread
From: Andrew Dunstan @ 2022-02-25 17:28 UTC (permalink / raw)
To: Andres Freund <[email protected]>; +Cc: Peter Eisentraut <[email protected]>; pgsql-hackers; Daniel Gustafsson <[email protected]>; Tom Lane <[email protected]>; Dagfinn Ilmari Mannsåker <[email protected]>
On 2/25/22 11:41, Andres Freund wrote:
> Hi,
>
> On 2022-02-25 09:43:20 -0500, Andrew Dunstan wrote:
>> AIUI TAP consumers are supposed to ignore lines they don't understand.
> Are they?
>
> In http://testanything.org/tap-version-13-specification.html there's:
>
> "Lines written to standard output matching /^(not )?ok\b/ must be interpreted
> as test lines. [...]All other lines must not be considered test output."
>
> That says that all other lines aren't "test ouput". But what does that mean?
> It certainly doesn't mean they can just be ignored, because obviously
> ^(TAP version|#|1..|Bail out) isn't to be ignored.
I don't think we're following spec 13.
>
> And then there's:
>
>
> "
> Anything else
>
> Any output that is not a version, a plan, a test line, a YAML block, a
> diagnostic or a bail out is incorrect. How a harness handles the incorrect
> line is undefined. Test::Harness silently ignores incorrect lines, but will
> become more stringent in the future. TAP::Harness reports TAP syntax errors at
> the end of a test run.
> "
>
> If I were to to implement a tap parser this wouldn't make me ignore lines.
>
>
> Contrasting to that:
> http://testanything.org/tap-specification.html
>
> "
> Anything else
>
> A TAP parser is required to not consider an unknown line as an error but may
> optionally choose to capture said line and hand it to the test harness,
> which may have custom behavior attached. This is to allow for forward
> compatability. Test::Harness silently ignores incorrect lines, but will
> become more stringent in the future. TAP::Harness reports TAP syntax errors
> at the end of a test run.
> "
>
> I honestly don't know what to make of that. Parsers are supposed to ignore
> unknown lines, Test::Harness silently ignores, but also "TAP::Harness reports
> TAP syntax errors"? This seems like a self contradictory mess.
>
I agree it's a mess. Both of these "specs" are incredibly loose. I guess
that happens when the spec comes after the implementation.
cheers
andrew
--
Andrew Dunstan
EDB: https://www.enterprisedb.com
^ permalink raw reply [nested|flat] 9+ messages in thread
* [PATCH v4 07/19] Prepare freeze tuples in heap_page_prune()
@ 2024-03-19 00:01 Melanie Plageman <[email protected]>
0 siblings, 0 replies; 9+ messages in thread
From: Melanie Plageman @ 2024-03-19 00:01 UTC (permalink / raw)
In order to combine the freeze and prune records, we must determine
which tuples are freezable before actually executing pruning. All of the
page modifications should be made in the same critical section along
with emitting the combined WAL. Determine whether or not tuples should
or must be frozen and whether or not the page will be all frozen as a
consequence during pruning.
---
src/backend/access/heap/pruneheap.c | 41 +++++++++++++++--
src/backend/access/heap/vacuumlazy.c | 68 +++++++---------------------
src/include/access/heapam.h | 12 +++++
3 files changed, 66 insertions(+), 55 deletions(-)
diff --git a/src/backend/access/heap/pruneheap.c b/src/backend/access/heap/pruneheap.c
index bd30296ef1a..afc5ea5e0e7 100644
--- a/src/backend/access/heap/pruneheap.c
+++ b/src/backend/access/heap/pruneheap.c
@@ -153,7 +153,7 @@ heap_page_prune_opt(Relation relation, Buffer buffer)
* not the relation has indexes, since we cannot safely determine
* that during on-access pruning with the current implementation.
*/
- heap_page_prune(relation, buffer, vistest, false,
+ heap_page_prune(relation, buffer, vistest, false, NULL,
&presult, NULL);
/*
@@ -206,6 +206,9 @@ heap_page_prune_opt(Relation relation, Buffer buffer)
* mark_unused_now indicates whether or not dead items can be set LP_UNUSED
* during pruning.
*
+ * pagefrz contains both input and output parameters used if the caller is
+ * interested in potentially freezing tuples on the page.
+ *
* presult contains output parameters needed by callers such as the number of
* tuples removed and the number of line pointers newly marked LP_DEAD.
* heap_page_prune() is responsible for initializing it.
@@ -217,6 +220,7 @@ void
heap_page_prune(Relation relation, Buffer buffer,
GlobalVisState *vistest,
bool mark_unused_now,
+ HeapPageFreeze *pagefrz,
PruneResult *presult,
OffsetNumber *off_loc)
{
@@ -247,11 +251,16 @@ heap_page_prune(Relation relation, Buffer buffer,
presult->ndeleted = 0;
presult->nnewlpdead = 0;
+ presult->nfrozen = 0;
/*
- * Keep track of whether or not the page is all_visible in case the caller
- * wants to use this information to update the VM.
+ * Caller will update the VM after pruning, collecting LP_DEAD items, and
+ * freezing tuples. Keep track of whether or not the page is all_visible
+ * and all_frozen and use this information to update the VM. all_visible
+ * implies lpdead_items == 0, but don't trust all_frozen result unless
+ * all_visible is also set to true.
*/
+ presult->all_frozen = true;
presult->all_visible = true;
/* for recovery conflicts */
presult->frz_conflict_horizon = InvalidTransactionId;
@@ -381,6 +390,32 @@ heap_page_prune(Relation relation, Buffer buffer,
elog(ERROR, "unexpected HeapTupleSatisfiesVacuum result");
break;
}
+
+ /*
+ * Consider freezing any normal tuples which will not be removed
+ */
+ if (presult->htsv[offnum] != HEAPTUPLE_DEAD && pagefrz)
+ {
+ bool totally_frozen;
+
+ /* Tuple with storage -- consider need to freeze */
+ if ((heap_prepare_freeze_tuple(htup, pagefrz,
+ &presult->frozen[presult->nfrozen],
+ &totally_frozen)))
+ {
+ /* Save prepared freeze plan for later */
+ presult->frozen[presult->nfrozen++].offset = offnum;
+ }
+
+ /*
+ * If any tuple isn't either totally frozen already or eligible to
+ * become totally frozen (according to its freeze plan), then the
+ * page definitely cannot be set all-frozen in the visibility map
+ * later on
+ */
+ if (!totally_frozen)
+ presult->all_frozen = false;
+ }
}
/*
diff --git a/src/backend/access/heap/vacuumlazy.c b/src/backend/access/heap/vacuumlazy.c
index 06e0e841582..4187c998d25 100644
--- a/src/backend/access/heap/vacuumlazy.c
+++ b/src/backend/access/heap/vacuumlazy.c
@@ -1416,16 +1416,13 @@ lazy_scan_prune(LVRelState *vacrel,
maxoff;
ItemId itemid;
PruneResult presult;
- int tuples_frozen,
- lpdead_items,
+ int lpdead_items,
live_tuples,
recently_dead_tuples;
HeapPageFreeze pagefrz;
bool hastup = false;
- bool all_frozen;
int64 fpi_before = pgWalUsage.wal_fpi;
OffsetNumber deadoffsets[MaxHeapTuplesPerPage];
- HeapTupleFreeze frozen[MaxHeapTuplesPerPage];
Assert(BufferGetBlockNumber(buf) == blkno);
@@ -1443,7 +1440,6 @@ lazy_scan_prune(LVRelState *vacrel,
pagefrz.NoFreezePageRelfrozenXid = vacrel->NewRelfrozenXid;
pagefrz.NoFreezePageRelminMxid = vacrel->NewRelminMxid;
pagefrz.cutoffs = &vacrel->cutoffs;
- tuples_frozen = 0;
lpdead_items = 0;
live_tuples = 0;
recently_dead_tuples = 0;
@@ -1461,31 +1457,20 @@ lazy_scan_prune(LVRelState *vacrel,
* false otherwise.
*/
heap_page_prune(rel, buf, vacrel->vistest, vacrel->nindexes == 0,
- &presult, &vacrel->offnum);
+ &pagefrz, &presult, &vacrel->offnum);
/*
* Now scan the page to collect LP_DEAD items and check for tuples
* requiring freezing among remaining tuples with storage. We will update
* the VM after collecting LP_DEAD items and freezing tuples. Pruning will
- * have determined whether or not the page is all_visible. Keep track of
- * whether or not the page is all_frozen and use this information to
- * update the VM. all_visible implies lpdead_items == 0, but don't trust
- * all_frozen result unless all_visible is also set to true.
+ * have determined whether or not the page is all_visible and able to
+ * become all_frozen.
*
*/
- all_frozen = true;
-
- /*
- * Now scan the page to collect LP_DEAD items and update the variables set
- * just above.
- */
for (offnum = FirstOffsetNumber;
offnum <= maxoff;
offnum = OffsetNumberNext(offnum))
{
- HeapTupleHeader htup;
- bool totally_frozen;
-
/*
* Set the offset number so that we can display it along with any
* error that occurred while processing this tuple.
@@ -1521,8 +1506,6 @@ lazy_scan_prune(LVRelState *vacrel,
Assert(ItemIdIsNormal(itemid));
- htup = (HeapTupleHeader) PageGetItem(page, itemid);
-
/*
* The criteria for counting a tuple as live in this block need to
* match what analyze.c's acquire_sample_rows() does, otherwise VACUUM
@@ -1587,29 +1570,8 @@ lazy_scan_prune(LVRelState *vacrel,
hastup = true; /* page makes rel truncation unsafe */
- /* Tuple with storage -- consider need to freeze */
- if (heap_prepare_freeze_tuple(htup, &pagefrz,
- &frozen[tuples_frozen], &totally_frozen))
- {
- /* Save prepared freeze plan for later */
- frozen[tuples_frozen++].offset = offnum;
- }
-
- /*
- * If any tuple isn't either totally frozen already or eligible to
- * become totally frozen (according to its freeze plan), then the page
- * definitely cannot be set all-frozen in the visibility map later on
- */
- if (!totally_frozen)
- all_frozen = false;
}
- /*
- * We have now divided every item on the page into either an LP_DEAD item
- * that will need to be vacuumed in indexes later, or a LP_NORMAL tuple
- * that remains and needs to be considered for freezing now (LP_UNUSED and
- * LP_REDIRECT items also remain, but are of no further interest to us).
- */
vacrel->offnum = InvalidOffsetNumber;
/*
@@ -1618,8 +1580,8 @@ lazy_scan_prune(LVRelState *vacrel,
* freeze when pruning generated an FPI, if doing so means that we set the
* page all-frozen afterwards (might not happen until final heap pass).
*/
- if (pagefrz.freeze_required || tuples_frozen == 0 ||
- (presult.all_visible_except_removable && all_frozen &&
+ if (pagefrz.freeze_required || presult.nfrozen == 0 ||
+ (presult.all_visible_except_removable && presult.all_frozen &&
fpi_before != pgWalUsage.wal_fpi))
{
/*
@@ -1629,7 +1591,7 @@ lazy_scan_prune(LVRelState *vacrel,
vacrel->NewRelfrozenXid = pagefrz.FreezePageRelfrozenXid;
vacrel->NewRelminMxid = pagefrz.FreezePageRelminMxid;
- if (tuples_frozen == 0)
+ if (presult.nfrozen == 0)
{
/*
* We have no freeze plans to execute, so there's no added cost
@@ -1657,7 +1619,7 @@ lazy_scan_prune(LVRelState *vacrel,
* once we're done with it. Otherwise we generate a conservative
* cutoff by stepping back from OldestXmin.
*/
- if (presult.all_visible_except_removable && all_frozen)
+ if (presult.all_visible_except_removable && presult.all_frozen)
{
/* Using same cutoff when setting VM is now unnecessary */
snapshotConflictHorizon = presult.frz_conflict_horizon;
@@ -1673,7 +1635,7 @@ lazy_scan_prune(LVRelState *vacrel,
/* Execute all freeze plans for page as a single atomic action */
heap_freeze_execute_prepared(vacrel->rel, buf,
snapshotConflictHorizon,
- frozen, tuples_frozen);
+ presult.frozen, presult.nfrozen);
}
}
else
@@ -1684,8 +1646,8 @@ lazy_scan_prune(LVRelState *vacrel,
*/
vacrel->NewRelfrozenXid = pagefrz.NoFreezePageRelfrozenXid;
vacrel->NewRelminMxid = pagefrz.NoFreezePageRelminMxid;
- all_frozen = false;
- tuples_frozen = 0; /* avoid miscounts in instrumentation */
+ presult.all_frozen = false;
+ presult.nfrozen = 0; /* avoid miscounts in instrumentation */
}
/*
@@ -1708,6 +1670,8 @@ lazy_scan_prune(LVRelState *vacrel,
&debug_cutoff, &debug_all_frozen))
Assert(false);
+ Assert(presult.all_frozen == debug_all_frozen);
+
Assert(!TransactionIdIsValid(debug_cutoff) ||
debug_cutoff == presult.frz_conflict_horizon);
}
@@ -1738,7 +1702,7 @@ lazy_scan_prune(LVRelState *vacrel,
/* Finally, add page-local counts to whole-VACUUM counts */
vacrel->tuples_deleted += presult.ndeleted;
- vacrel->tuples_frozen += tuples_frozen;
+ vacrel->tuples_frozen += presult.nfrozen;
vacrel->lpdead_items += lpdead_items;
vacrel->live_tuples += live_tuples;
vacrel->recently_dead_tuples += recently_dead_tuples;
@@ -1761,7 +1725,7 @@ lazy_scan_prune(LVRelState *vacrel,
{
uint8 flags = VISIBILITYMAP_ALL_VISIBLE;
- if (all_frozen)
+ if (presult.all_frozen)
{
Assert(!TransactionIdIsValid(presult.frz_conflict_horizon));
flags |= VISIBILITYMAP_ALL_FROZEN;
@@ -1832,7 +1796,7 @@ lazy_scan_prune(LVRelState *vacrel,
* true, so we must check both all_visible and all_frozen.
*/
else if (all_visible_according_to_vm && presult.all_visible &&
- all_frozen && !VM_ALL_FROZEN(vacrel->rel, blkno, &vmbuffer))
+ presult.all_frozen && !VM_ALL_FROZEN(vacrel->rel, blkno, &vmbuffer))
{
/*
* Avoid relying on all_visible_according_to_vm as a proxy for the
diff --git a/src/include/access/heapam.h b/src/include/access/heapam.h
index 297ba03bf09..2339abfd28a 100644
--- a/src/include/access/heapam.h
+++ b/src/include/access/heapam.h
@@ -201,6 +201,11 @@ typedef struct PruneResult
int nnewlpdead; /* Number of newly LP_DEAD items */
bool all_visible; /* Whether or not the page is all visible */
bool all_visible_except_removable;
+ /* Whether or not the page can be set all frozen in the VM */
+ bool all_frozen;
+
+ /* Number of newly frozen tuples */
+ int nfrozen;
TransactionId frz_conflict_horizon; /* Newest xmin on the page */
/*
@@ -213,6 +218,12 @@ typedef struct PruneResult
* 1. Otherwise every access would need to subtract 1.
*/
int8 htsv[MaxHeapTuplesPerPage + 1];
+
+
+ /*
+ * One entry for every tuple that we may freeze.
+ */
+ HeapTupleFreeze frozen[MaxHeapTuplesPerPage];
} PruneResult;
/*
@@ -324,6 +335,7 @@ extern void heap_page_prune_opt(Relation relation, Buffer buffer);
extern void heap_page_prune(Relation relation, Buffer buffer,
struct GlobalVisState *vistest,
bool mark_unused_now,
+ HeapPageFreeze *pagefrz,
PruneResult *presult,
OffsetNumber *off_loc);
extern void heap_page_prune_execute(Buffer buffer,
--
2.40.1
--tez7m2a73jtztiij
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v4-0008-lazy_scan_prune-reorder-freeze-execution-logic.patch"
^ permalink raw reply [nested|flat] 9+ messages in thread
end of thread, other threads:[~2024-03-19 00:01 UTC | newest]
Thread overview: 9+ messages (download: mbox mbox.gz follow: Atom feed)
-- links below jump to the message on this page --
2017-12-26 08:43 [PATCH 1/4] Remove entries that haven't been used for a certain time Kyotaro Horiguchi <[email protected]>
2017-12-26 08:43 [PATCH 1/3] Remove entries that haven't been used for a certain time Kyotaro Horiguchi <[email protected]>
2018-10-16 04:04 [PATCH 1/3] Remove entries that haven't been used for a certain time Kyotaro Horiguchi <[email protected]>
2022-02-25 14:43 Re: Readd use of TAP subtests Andrew Dunstan <[email protected]>
2022-02-25 16:35 ` Re: Readd use of TAP subtests Jacob Champion <[email protected]>
2022-02-25 16:38 ` Re: Readd use of TAP subtests Jacob Champion <[email protected]>
2022-02-25 16:41 ` Re: Readd use of TAP subtests Andres Freund <[email protected]>
2022-02-25 17:28 ` Re: Readd use of TAP subtests Andrew Dunstan <[email protected]>
2024-03-19 00:01 [PATCH v4 07/19] Prepare freeze tuples in heap_page_prune() Melanie Plageman <[email protected]>
This inbox is served by agora; see mirroring instructions
for how to clone and mirror all data and code used for this inbox