pgpool2.git / summary / log / commit / refs
commit 91297c7cb841fa7a3fc0909de7565ce81301b98f
Author: Tatsuo Ishii <ishii at sraoss.co.jp>
Date: Wed Sep 14 13:15:38 2011 +0000
Add query cache stats.
Fix unable to compile without memcached.
child.c | 2 +-
main.c | 6 ++
pool.h | 3 +-
pool_memqcache.c | 202 +++++++++++++++++++++++++++++++++++++++--------
pool_memqcache.h | 32 ++++++++
pool_process_reporting.c | 15 ++++
pool_proto_modules.c | 1 +
pool_session_context.c | 6 ++
pool_session_context.h | 3 +-
9 files changed, 236 insertions(+), 34 deletions(-)
diff --git a/child.c b/child.c
index 2444bb128..1c31608c7 100644
--- a/child.c
+++ b/child.c
@@ -1396,7 +1396,7 @@ void child_exit(int code)
pool_close(pool_system_db_connection()->con);
}
- if (pool_config->memory_cache_enabled && !pool_is_shmem_cache)
+ if (pool_config->memory_cache_enabled && !pool_is_shmem_cache())
{
memcached_disconnect();
}
diff --git a/main.c b/main.c
index d65661721..642b01767 100644
--- a/main.c
+++ b/main.c
@@ -566,6 +566,12 @@ int main(int argc, char **argv)
pool_allocate_fsmm_clock_hand();
+ if (pool_init_memqcache_stats() < 0)
+ {
+ pool_error("pool_init_memqcache_stats error");
+ myexit(1);
+ }
+
pool_discard_oid_maps();
}
diff --git a/pool.h b/pool.h
index 620ac8fb3..f09aee364 100644
--- a/pool.h
+++ b/pool.h
@@ -329,10 +329,11 @@ extern BACKEND_STATUS* my_backend_status[];
#define NO_LOAD_BALANCE "/*NO LOAD BALANCE*/"
#define NO_LOAD_BALANCE_COMMENT_SZ (sizeof(NO_LOAD_BALANCE)-1)
-#define MAX_NUM_SEMAPHORES 3
+#define MAX_NUM_SEMAPHORES 4
#define CONN_COUNTER_SEM 0
#define REQUEST_INFO_SEM 1
#define SHM_CACHE_SEM 2
+#define QUERY_CACHE_STATS_SEM 3
/*
* number specified when semaphore is locked/unlocked
diff --git a/pool_memqcache.c b/pool_memqcache.c
index 555dfd5bb..81f6b4b09 100644
--- a/pool_memqcache.c
+++ b/pool_memqcache.c
@@ -55,12 +55,16 @@ memcached_st *memc;
#endif
static char* encode_key(const char *s, char *buf, POOL_CONNECTION_POOL *backend);
+#ifdef DEBU
static void dump_cache_data(const char *data, size_t len);
+#endif
static int pool_commit_cache(POOL_CONNECTION_POOL *backend, char *query, char *data, size_t datalen, int num_oids, int *oids);
static int pool_fetch_cache(POOL_CONNECTION_POOL *backend, const char *query, char **buf, size_t *len);
static int send_cached_messages(POOL_CONNECTION *frontend, const char *qcache, int qcachelen);
static void send_message(POOL_CONNECTION *conn, char kind, int len, const char *data);
+#ifdef USE_MEMCACHED
static int delete_cache_on_memcached(const char *key);
+#endif
static int pool_get_dml_table_oid(int **oid);
static void pool_discard_dml_table_oid(void);
static void pool_invalidate_query_cache(int num_table_oids, int *table_oid, bool unlink);
@@ -98,7 +102,9 @@ static char *block_address(int blockid);
static POOL_CACHE_ITEM_POINTER *item_pointer(char *block, int i);
static POOL_CACHE_ITEM_HEADER *item_header(char *block, int i);
static POOL_CACHE_BLOCKID pool_reuse_block(void);
+#ifdef SHMEMCACHE_DEBUG
static void dump_shmem_cache(POOL_CACHE_BLOCKID blockid);
+#endif
/*
* Connect to Memcached
@@ -110,13 +116,13 @@ int memcached_connect (void)
#ifdef USE_MEMCACHED
memcached_server_st *servers;
memcached_return rc;
-#endif
/* Already connected? */
if (memc)
{
return 0;
}
+#endif
memqcache_memcached_host = pool_config->memqcache_memcached_host;
memqcache_memcached_port = pool_config->memqcache_memcached_port;
@@ -150,12 +156,11 @@ int memcached_connect (void)
*/
void memcached_disconnect (void)
{
+#ifdef USE_MEMCACHED
if (!memc)
{
return;
}
-
-#ifdef USE_MEMCACHED
memcached_free(memc);
#else
pool_error("memcached_disconnect: memcached support is not enabled");
@@ -395,6 +400,7 @@ static char* encode_key(const char *s, char *buf, POOL_CONNECTION_POOL *backend)
return buf;
}
+#ifdef DEBUG
/*
* dump cache data
*/
@@ -426,6 +432,7 @@ static void dump_cache_data(const char *data, size_t len)
fprintf(stderr, "\n");
}
}
+#endif
/*
* send cached messages
@@ -494,18 +501,18 @@ static void send_message(POOL_CONNECTION *conn, char kind, int len, const char *
pool_write(conn, (void *)data, len-sizeof(len));
}
+#ifdef USE_MEMCACHED
/*
* delete query cache on memcached
*/
static int delete_cache_on_memcached(const char *key)
{
-#ifdef USE_MEMCACHED
+
memcached_return rc;
-#endif
pool_debug("delete_cache_on_memcached: key: %s", key);
-#ifdef USE_MEMCACHED
+
/* delete cache data on memcached. key is md5 hash query */
rc= memcached_delete(memc, key, 32, (time_t)0);
@@ -516,12 +523,9 @@ static int delete_cache_on_memcached(const char *key)
return 0;
}
return 1;
-#else
- pool_error("delete_cache_on_memcached: memecahced support is not enabled");
- return 0;
-#endif
-}
+}
+#endif
/*
* Fetch SELECT data from cache if possible.
@@ -1181,6 +1185,7 @@ static void pool_reset_memqcache_buffer(void)
session_context->query_cache_array = pool_create_query_cache_array();
}
pool_discard_dml_table_oid();
+ pool_tmp_stats_reset_num_selects();
}
/*
@@ -2007,6 +2012,7 @@ static POOL_CACHE_ITEM_HEADER *item_header(char *block, int i)
return (POOL_CACHE_ITEM_HEADER *)(block + cip->offset);
}
+#ifdef SHMEMCACHE_DEBUG
/*
* Dump shmem cache block
*/
@@ -2030,6 +2036,7 @@ static void dump_shmem_cache(POOL_CACHE_BLOCKID blockid)
blockid, i, sizeof(*cih), cih->timestamp, cih->total_length);
}
}
+#endif
/*
* SELECT query result array modules
@@ -2450,6 +2457,11 @@ void pool_handle_query_cache(POOL_CONNECTION_POOL *backend, char *query, Node *n
free(cache_buffer);
}
pool_shmem_unlock();
+
+ /* Count up SELECT stats */
+ pool_stats_count_up_num_selects(1);
+
+ /* Reset temp buffer */
pool_reset_memqcache_buffer();
}
else
@@ -2459,6 +2471,9 @@ void pool_handle_query_cache(POOL_CONNECTION_POOL *backend, char *query, Node *n
/* In transaction. Keep to temp query cache array */
pool_add_oids_temp_query_cache(cache, num_oids, oids);
pool_add_query_cache_array(session_context->query_cache_array, cache);
+
+ /* Count up temporary SELECT stats */
+ pool_tmp_stats_count_up_num_selects();
}
}
else if (is_rollback_query(node)) /* Rollback? */
@@ -2503,39 +2518,164 @@ void pool_handle_query_cache(POOL_CONNECTION_POOL *backend, char *query, Node *n
free(cache_buffer);
}
pool_shmem_unlock();
+
+ /* Count up number of SELECT stats */
+ pool_stats_count_up_num_selects(pool_tmp_stats_get_num_selects());
+
pool_reset_memqcache_buffer();
}
else /* Non cache safe queries */
{
- /*
- * DML/DCL/DDL/non cache safe SELECT case
- */
-
- /* Extract table oids from buffer */
- num_oids = pool_get_dml_table_oid(&oids);
- if (num_oids > 0)
+ /* Non cachable SELECT */
+ if (IsA(node, SelectStmt))
{
- /*
- * If we are not inside a transaction, we can
- * immediately invalidate query cache.
- */
if (state == 'I')
{
- pool_shmem_lock();
- pool_invalidate_query_cache(num_oids, oids, true);
- pool_shmem_unlock();
+ /* Count up SELECT stats */
+ pool_stats_count_up_num_selects(1);
pool_reset_memqcache_buffer();
}
else
+ {
+ /* Count up temporary SELECT stats */
+ pool_tmp_stats_count_up_num_selects();
+ }
+ }
+ else
+ {
+ /*
+ * DML/DCL/DDL case
+ */
+
+ /* Extract table oids from buffer */
+ num_oids = pool_get_dml_table_oid(&oids);
+ if (num_oids > 0)
{
/*
- * If we are inside a transaction, we
- * cannot invalidate query cache
- * yet. However we can clear cache buffer,
- * if DML/DDL modifies the TABLE which SELECT uses.
+ * If we are not inside a transaction, we can
+ * immediately invalidate query cache.
*/
- pool_check_and_discard_cache_buffer(num_oids, oids);
- }
+ if (state == 'I')
+ {
+ pool_shmem_lock();
+ pool_invalidate_query_cache(num_oids, oids, true);
+ pool_shmem_unlock();
+ pool_reset_memqcache_buffer();
+ }
+ else
+ {
+ /*
+ * If we are inside a transaction, we
+ * cannot invalidate query cache
+ * yet. However we can clear cache buffer,
+ * if DML/DDL modifies the TABLE which SELECT uses.
+ */
+ pool_check_and_discard_cache_buffer(num_oids, oids);
+ }
+ }
}
}
}
+
+/*
+ * Create and initialize query cache stats
+ */
+static POOL_QUERY_CACHE_STATS *stats;
+int pool_init_memqcache_stats(void)
+{
+ stats = pool_shared_memory_create(sizeof(POOL_QUERY_CACHE_STATS));
+ if (stats == NULL)
+ {
+ pool_error("pool_init_meqcache_stats: failed to allocate shared memory stats. request size: %zd",
+ sizeof(POOL_QUERY_CACHE_STATS));
+ return -1;
+ }
+
+ pool_reset_memqcache_stats();
+
+ return 0;
+}
+
+/*
+ * Returns copy of stats area. The copy is in static area and will be
+ * overwritten by next call to this function.
+ */
+POOL_QUERY_CACHE_STATS *pool_get_memqcache_stats(void)
+{
+ static POOL_QUERY_CACHE_STATS mystats;
+
+ pool_semaphore_lock(QUERY_CACHE_STATS_SEM);
+ memcpy(&mystats, stats, sizeof(POOL_QUERY_CACHE_STATS));
+ pool_semaphore_unlock(QUERY_CACHE_STATS_SEM);
+
+ return &mystats;
+}
+
+/*
+ * Reset query cache stats. Caller must lock QUERY_CACHE_STATS_SEM if
+ * neccessary.
+ */
+void pool_reset_memqcache_stats(void)
+{
+ memset(stats, 0, sizeof(POOL_QUERY_CACHE_STATS));
+ stats->start_time = time(NULL);
+}
+
+/*
+ * Count up number of successfull SELECTs and returns the number.
+ * QUERY_CACHE_STATS_SEM lock is aquired in this function.
+ */
+long long int pool_stats_count_up_num_selects(long long int num)
+{
+ pool_semaphore_lock(QUERY_CACHE_STATS_SEM);
+ stats->num_selects += num;
+ pool_semaphore_unlock(QUERY_CACHE_STATS_SEM);
+ return stats->num_selects;
+}
+
+/*
+ * Count up number of successfull SELECTs in temporary area and returns
+ * the number.
+ */
+long long int pool_tmp_stats_count_up_num_selects(void)
+{
+ POOL_SESSION_CONTEXT *session_context;
+
+ session_context = pool_get_session_context();
+ session_context->num_selects++;
+ return session_context->num_selects;
+}
+
+/*
+ * Return number of successfull SELECTs in temporary area.
+ */
+long long int pool_tmp_stats_get_num_selects(void)
+{
+ POOL_SESSION_CONTEXT *session_context;
+
+ session_context = pool_get_session_context();
+ return session_context->num_selects;
+}
+
+/*
+ * Reset number of successfull SELECTs in temporary area.
+ */
+void pool_tmp_stats_reset_num_selects(void)
+{
+ POOL_SESSION_CONTEXT *session_context;
+
+ session_context = pool_get_session_context();
+ session_context->num_selects = 0;
+}
+
+/*
+ * Count up number of SELECTs extracted from cache returns the number.
+ * QUERY_CACHE_STATS_SEM lock is aquired in this function.
+ */
+long long int pool_stats_count_up_num_cache_hits(void)
+{
+ pool_semaphore_lock(QUERY_CACHE_STATS_SEM);
+ stats->num_cache_hits++;
+ pool_semaphore_unlock(QUERY_CACHE_STATS_SEM);
+ return stats->num_cache_hits;
+}
diff --git a/pool_memqcache.h b/pool_memqcache.h
index 55c82e8bd..91b736b36 100644
--- a/pool_memqcache.h
+++ b/pool_memqcache.h
@@ -114,12 +114,18 @@ extern int memcached_connect(void);
extern void memcached_disconnect(void);
extern void memqcache_register(char kind, POOL_CONNECTION *frontend, char *data, int data_len);
+/*
+ * Cache key
+ */
typedef union
{
POOL_CACHEID cacheid; /* cache key (shmem configuration) */
char hashkey[32]; /* cache key (memcached configuration) */
} POOL_CACHEKEY;
+/*
+ * Internal buffer structure
+ */
typedef struct
{
size_t bufsize; /* buffer size */
@@ -127,6 +133,9 @@ typedef struct
char *buf; /* buffer */
} POOL_INTERNAL_BUFFER;
+/*
+ * Temporary query cache buffer
+ */
typedef struct
{
bool is_exceeded; /* true if data size exceeds memqcache_maxcache */
@@ -137,6 +146,9 @@ typedef struct
POOL_INTERNAL_BUFFER *oids;
} POOL_TEMP_QUERY_CACHE;
+/*
+ * Temporary query cache buffer array
+ */
typedef struct
{
int num_caches;
@@ -144,6 +156,17 @@ typedef struct
POOL_TEMP_QUERY_CACHE *caches[1]; /* actual data continues... */
} POOL_QUERY_CACHE_ARRAY;
+/*
+ * Query cache statistics structure. This area must be placed on shared
+ * memory and protected by QUERY_CACHE_STATS_SEM.
+ */
+typedef struct
+{
+ time_t start_time; /* start time when the statistics begins */
+ long long int num_selects; /* number of successfull SELECTs */
+ long long int num_cache_hits; /* number of SELECTs extracted from cache */
+} POOL_QUERY_CACHE_STATS;
+
extern POOL_STATUS pool_fetch_from_memory_cache(POOL_CONNECTION *frontend,
POOL_CONNECTION_POOL *backend,
char *contents, bool *foundp);
@@ -166,4 +189,13 @@ extern void pool_discard_query_cache_array(POOL_QUERY_CACHE_ARRAY *cache_array);
extern POOL_TEMP_QUERY_CACHE *pool_create_temp_query_cache(char *query);
extern void pool_handle_query_cache(POOL_CONNECTION_POOL *backend, char *query, Node *node, char state);
+extern int pool_init_memqcache_stats(void);
+extern POOL_QUERY_CACHE_STATS *pool_get_memqcache_stats(void);
+extern void pool_reset_memqcache_stats(void);
+extern long long int pool_stats_count_up_num_selects(long long int num);
+extern long long int pool_stats_count_up_num_cache_hits(void);
+extern long long int pool_tmp_stats_count_up_num_selects(void);
+extern long long int pool_tmp_stats_get_num_selects(void);
+extern void pool_tmp_stats_reset_num_selects(void);
+
#endif /* POOL_MEMQCACHE_H */
diff --git a/pool_process_reporting.c b/pool_process_reporting.c
index 49c89048d..da73fced8 100644
--- a/pool_process_reporting.c
+++ b/pool_process_reporting.c
@@ -579,6 +579,21 @@ POOL_REPORT_CONFIG* get_config(int *nrows)
strncpy(status[i].desc, "Tempory work directory to record table oids", POOLCONFIG_MAXDESCLEN);
i++;
+ strncpy(status[i].name, "memqcache_stats_start_time", POOLCONFIG_MAXNAMELEN);
+ snprintf(status[i].value, POOLCONFIG_MAXVALLEN, "%s", ctime(&pool_get_memqcache_stats()->start_time));
+ strncpy(status[i].desc, "Start time of query cache stats", POOLCONFIG_MAXDESCLEN);
+ i++;
+
+ strncpy(status[i].name, "memqcache_no_cache_hits", POOLCONFIG_MAXNAMELEN);
+ snprintf(status[i].value, POOLCONFIG_MAXVALLEN, "%lld", pool_get_memqcache_stats()->num_selects);
+ strncpy(status[i].desc, "Number of SELECTs not hitting query cache", POOLCONFIG_MAXDESCLEN);
+ i++;
+
+ strncpy(status[i].name, "memqcache_cache_hits", POOLCONFIG_MAXNAMELEN);
+ snprintf(status[i].value, POOLCONFIG_MAXVALLEN, "%lld", pool_get_memqcache_stats()->num_cache_hits);
+ strncpy(status[i].desc, "Number of SELECTs hitting query cache", POOLCONFIG_MAXDESCLEN);
+ i++;
+
for (j = 0; j < NUM_BACKENDS; j++)
{
if (BACKEND_INFO(j).backend_port == 0)
diff --git a/pool_proto_modules.c b/pool_proto_modules.c
index 92665f1ea..67df3f947 100644
--- a/pool_proto_modules.c
+++ b/pool_proto_modules.c
@@ -165,6 +165,7 @@ POOL_STATUS SimpleQuery(POOL_CONNECTION *frontend,
if (foundp)
{
pool_set_skip_reading_from_backends();
+ pool_stats_count_up_num_cache_hits();
return POOL_CONTINUE;
}
}
diff --git a/pool_session_context.c b/pool_session_context.c
index e3a2015f1..c1612d3e2 100644
--- a/pool_session_context.c
+++ b/pool_session_context.c
@@ -114,7 +114,10 @@ void pool_init_session_context(POOL_CONNECTION *frontend, POOL_CONNECTION_POOL *
session_context->mismatch_ntuples = false;
if (pool_config->memory_cache_enabled)
+ {
session_context->query_cache_array = pool_create_query_cache_array();
+ session_context->num_selects = 0;
+ }
}
/*
@@ -128,7 +131,10 @@ void pool_session_context_destroy(void)
free(session_context->message_list.sent_messages);
pool_memory_delete(session_context->memory_context, 0);
if (pool_config->memory_cache_enabled)
+ {
pool_discard_query_cache_array(session_context->query_cache_array);
+ session_context->num_selects = 0;
+ }
}
/* XXX For now, just zap memory */
memset(&session_context_d, 0, sizeof(session_context_d));
diff --git a/pool_session_context.h b/pool_session_context.h
index 6debe8c8a..9265d26f4 100644
--- a/pool_session_context.h
+++ b/pool_session_context.h
@@ -147,7 +147,8 @@ typedef struct {
/*
* Query cache management area
*/
- POOL_QUERY_CACHE_ARRAY *query_cache_array;
+ POOL_QUERY_CACHE_ARRAY *query_cache_array; /* pending SELECT results */
+ long long int num_selects; /* number of successfull SELECTs in this transaction */
} POOL_SESSION_CONTEXT;
extern void pool_init_session_context(POOL_CONNECTION *frontend, POOL_CONNECTION_POOL *backend);
[parent: 8a901ec6d7e1]