pgpool2.git / summary / log / commit / refs
commit ba02ccd0cb8f189182d67fc3666b8c13bf45c9e3
Author: Yoshiyuki Asaba <y-asaba at pgfoundry.org>
Date: Wed Sep 26 05:29:47 2007 +0000
pgpool-II could not detach multiple node when message kinds were
mismatched.
For example:
node-1(master): success
node-2: fail
node-3: success
node-4: fail
node-5: success
node-2 was only detached.
Now it determine detached nodes by "majority voting rule".
And it can detach them at the same time.
For example:
node-1(master): success
node-2: fail
node-3: success
node-4: fail
node-5: success
node-2 and node-4 are detached because they are minority.
node-1(master): success
node-2: fail
node-3: fail
node-4: success
The number of the group which gain "success" is same as the number of
the group which gain "fail". In this case, node-2 and node-3 are
detached because these don't belong to the group which master belongs
to.
main.c | 121 +++++++++++++++++++++++++++++++--------------------
pool.h | 4 +-
pool_process_query.c | 96 +++++++++++++++++++++++++++++-----------
pool_stream.c | 14 +++---
4 files changed, 153 insertions(+), 82 deletions(-)
diff --git a/main.c b/main.c
index 3340eb4c2..4fc51edbe 100644
--- a/main.c
+++ b/main.c
@@ -401,7 +401,7 @@ int main(int argc, char **argv)
/* initialize Req_info */
Req_info->kind = NODE_UP_REQUEST;
- Req_info->node_id = -1;
+ memset(Req_info->node_id, -1, sizeof(int) * MAX_NUM_BACKENDS);
Req_info->master_node_id = 0;
Req_info->conn_counter = 0;
@@ -504,7 +504,7 @@ int main(int argc, char **argv)
{
pool_log("set %d th backend down status", sts);
Req_info->kind = NODE_DOWN_REQUEST;
- Req_info->node_id = sts;
+ Req_info->node_id[0] = sts;
failover();
/* need to distribute this info to children */
}
@@ -518,7 +518,7 @@ int main(int argc, char **argv)
/* retry count over */
pool_log("set %d th backend down status", sts);
Req_info->kind = NODE_DOWN_REQUEST;
- Req_info->node_id = sts;
+ Req_info->node_id[0] = sts;
failover();
retrycnt = 0;
}
@@ -925,28 +925,40 @@ static void myexit(int code)
exit(code);
}
-/* notice backend connection error using SIGUSR1 */
void notice_backend_error(int node_id)
+{
+ int n = node_id;
+
+ degenerate_backend_set(&n, 1);
+}
+
+/* notice backend connection error using SIGUSR1 */
+void degenerate_backend_set(int *node_id_set, int count)
{
pid_t parent = getppid();
+ int i;
if (pool_config->parallel_mode)
{
return;
}
- pool_log("notice_backend_error: %d fail over request from pid %d", node_id, getpid());
-
+ pool_semaphore_lock(REQUEST_INFO_SEM);
Req_info->kind = NODE_DOWN_REQUEST;
- Req_info->node_id = node_id;
-
- if (node_id < 0 || node_id >= MAX_NUM_BACKENDS || !VALID_BACKEND(node_id))
+ for (i = 0; i < count; i++)
{
- pool_error("notice_backend_error: node %d is not valid backend.");
- return;
- }
+ if (node_id_set[i] < 0 || node_id_set[i] >= MAX_NUM_BACKENDS ||
+ !VALID_BACKEND(node_id_set[i]))
+ {
+ pool_log("notice_backend_error: node %d is not valid backend.");
+ continue;
+ }
+ pool_log("notice_backend_error: %d fail over request from pid %d", node_id_set[i], getpid());
+ Req_info->node_id[i] = node_id_set[i];
+ }
kill(parent, SIGUSR1);
+ pool_semaphore_unlock(REQUEST_INFO_SEM);
}
/* send failback request using SIGUSR1 */
@@ -956,7 +968,7 @@ void send_failback_request(int node_id)
pool_log("send_failback_request: fail back %d th node request from pid %d", node_id, getpid());
Req_info->kind = NODE_UP_REQUEST;
- Req_info->node_id = node_id;
+ Req_info->node_id[0] = node_id;
if (node_id < 0 || node_id >= MAX_NUM_BACKENDS || VALID_BACKEND(node_id))
{
@@ -1078,45 +1090,16 @@ static void failover(void)
return;
}
+ pool_semaphore_lock(REQUEST_INFO_SEM);
+
if (Req_info->kind == CLOSE_IDLE_REQUEST)
{
+ pool_semaphore_unlock(REQUEST_INFO_SEM);
kill_all_children(SIGUSR1);
kill(pcp_pid, SIGUSR2);
return;
}
- node_id = Req_info->node_id;
-
- if (node_id < 0)
- {
- pool_error("failover_handler: invalid node_id %d", node_id);
- kill(pcp_pid, SIGUSR2);
- return;
- }
-
- if (node_id >= MAX_NUM_BACKENDS ||
- (Req_info->kind == NODE_UP_REQUEST && VALID_BACKEND(node_id)) ||
- (Req_info->kind == NODE_DOWN_REQUEST && !VALID_BACKEND(node_id)))
- {
- pool_error("failover_handler: invalid node_id %d status:%d MAX_NUM_BACKENDS: %d", node_id,
- BACKEND_INFO(node_id).backend_status, MAX_NUM_BACKENDS);
- kill(pcp_pid, SIGUSR2);
- return;
- }
-
- if (Req_info->kind == NODE_UP_REQUEST)
- {
- pool_log("starting fail back. reconnect host %s(%d)",
- BACKEND_INFO(node_id).backend_hostname,
- BACKEND_INFO(node_id).backend_port);
- }
- else
- {
- pool_log("starting degeneration. shutdown host %s(%d)",
- BACKEND_INFO(node_id).backend_hostname,
- BACKEND_INFO(node_id).backend_port);
- }
-
/*
* if not in replication mode/master slave mode, we treat this a restart request.
* otherwise we need to check if we have already failovered.
@@ -1127,13 +1110,51 @@ static void failover(void)
/* failback request? */
if (Req_info->kind == NODE_UP_REQUEST)
{
+ node_id = Req_info->node_id[0];
+ if (node_id >= MAX_NUM_BACKENDS ||
+ (Req_info->kind == NODE_UP_REQUEST && VALID_BACKEND(node_id)) ||
+ (Req_info->kind == NODE_DOWN_REQUEST && !VALID_BACKEND(node_id)))
+ {
+ pool_semaphore_unlock(REQUEST_INFO_SEM);
+ pool_error("failover_handler: invalid node_id %d status:%d MAX_NUM_BACKENDS: %d", node_id,
+ BACKEND_INFO(node_id).backend_status, MAX_NUM_BACKENDS);
+ kill(pcp_pid, SIGUSR2);
+ return;
+ }
+
+ pool_log("starting fail back. reconnect host %s(%d)",
+ BACKEND_INFO(node_id).backend_hostname,
+ BACKEND_INFO(node_id).backend_port);
BACKEND_INFO(node_id).backend_status = CON_CONNECT_WAIT; /* unset down status */
trigger_failover_command(node_id, pool_config->failback_command);
}
else
{
- BACKEND_INFO(node_id).backend_status = CON_DOWN; /* set down status */
- trigger_failover_command(node_id, pool_config->failover_command);
+ int cnt = 0;
+
+ for (i = 0; i < MAX_NUM_BACKENDS; i++)
+ {
+ if (Req_info->node_id[i] != -1 &&
+ VALID_BACKEND(Req_info->node_id[i]))
+ {
+ pool_log("starting degeneration. shutdown host %s(%d)",
+ BACKEND_INFO(Req_info->node_id[i]).backend_hostname,
+ BACKEND_INFO(Req_info->node_id[i]).backend_port);
+
+
+ BACKEND_INFO(Req_info->node_id[i]).backend_status = CON_DOWN; /* set down status */
+ trigger_failover_command(node_id, pool_config->failover_command);
+ cnt++;
+ }
+ }
+
+ if (cnt == 0)
+ {
+ pool_log("failover: no backends are degenerated");
+ pool_semaphore_unlock(REQUEST_INFO_SEM);
+ kill(pcp_pid, SIGUSR2);
+ return;
+ }
}
for (i=0;i<pool_config->backend_desc->num_backends;i++)
@@ -1164,6 +1185,7 @@ static void failover(void)
BACKEND_INFO(node_id).backend_port);
}
+ pool_semaphore_unlock(REQUEST_INFO_SEM);
switching = 0;
kill(pcp_pid, SIGUSR2);
return;
@@ -1193,6 +1215,9 @@ static void failover(void)
pool_error("failover_handler: wait() failed. reason:%s", strerror(errno));
#endif
+ memset(Req_info->node_id, -1, sizeof(int) * MAX_NUM_BACKENDS);
+ pool_semaphore_unlock(REQUEST_INFO_SEM);
+
/* fork the children */
for (i=0;i<pool_config->num_init_children;i++)
{
@@ -1213,7 +1238,6 @@ static void failover(void)
BACKEND_INFO(node_id).backend_port);
}
- Req_info->node_id = -1;
switching = 0;
kill(pcp_pid, SIGUSR2);
}
@@ -1728,6 +1752,7 @@ static int trigger_failover_command(int node, const char *command_line)
}
pool_memory_delete(pool_memory, 0);
+ pool_memory = NULL;
return r;
}
diff --git a/pool.h b/pool.h
index 4165e9016..11cf96f15 100644
--- a/pool.h
+++ b/pool.h
@@ -343,6 +343,7 @@ typedef struct {
#define MAX_NUM_SEMAPHORES 3
#define CONN_COUNTER_SEM 0
+#define REQUEST_INFO_SEM 1
#define MY_PROCESS_INFO (pids[my_proc_id])
@@ -368,7 +369,7 @@ typedef enum {
typedef struct {
POOL_REQUEST_KIND kind; /* request kind */
- int node_id; /* request node id */
+ int node_id[MAX_NUM_BACKENDS]; /* request node id */
int master_node_id; /* the youngest node id which is not in down status */
int conn_counter;
} POOL_REQUEST_INFO;
@@ -438,6 +439,7 @@ extern POOL_STATUS NoticeResponse(POOL_CONNECTION *frontend,
POOL_CONNECTION_POOL *backend);
extern void notice_backend_error(int node_id);
+extern void degenerate_backend_set(int *node_id_set, int count);
extern void send_failback_request(int node_id);
extern void pool_connection_pool_timer(POOL_CONNECTION_POOL *backend);
diff --git a/pool_process_query.c b/pool_process_query.c
index b2cd1b592..4494c09a8 100644
--- a/pool_process_query.c
+++ b/pool_process_query.c
@@ -3281,7 +3281,6 @@ static void process_reporting(POOL_CONNECTION *frontend, POOL_CONNECTION_POOL *b
/* data row */
for (i=0;i<nrows;i++)
{
- pool_debug("XXX %d", i);
pool_write(frontend, "D", 1);
len = sizeof(len) + sizeof(nrows);
len += sizeof(int) + strlen(status[i].name);
@@ -4648,46 +4647,91 @@ static POOL_STATUS read_kind_from_one_backend(POOL_CONNECTION *frontend, POOL_CO
static POOL_STATUS read_kind_from_backend(POOL_CONNECTION *frontend, POOL_CONNECTION_POOL *backend, char *kind)
{
int i;
+ char kind_list[MAX_NUM_BACKENDS];
+ char kind_map[256]; /* 256 is the number of sizeof(char) */
+ int max_kind = 0;
+ double max_count = 0;
+ int degenerate_node_num = 0;
+ int degenerate_node[MAX_NUM_BACKENDS];
+
+ memset(kind_list, -1, sizeof(kind_list));
+ memset(kind_map, 0, sizeof(kind_map));
for (i=0;i<NUM_BACKENDS;i++)
{
if (VALID_BACKEND(i))
{
- char k;
-
- if (pool_read(CONNECTION(backend, i), &k, 1) < 0)
+ if (pool_read(CONNECTION(backend, i), &kind_list[i], 1) < 0)
{
pool_error("pool_process_query: failed to read kind from %d th backend", i);
return POOL_ERROR;
}
- pool_debug("read_kind_from_backend: read kind from %d th backend %c NUM_BACKENDS: %d", i, k, NUM_BACKENDS);
+ pool_debug("read_kind_from_backend: read kind from %d th backend %c NUM_BACKENDS: %d", i, kind_list[i], NUM_BACKENDS);
+ }
+ }
- if (IS_MASTER_NODE_ID(i))
+ /* register kind map */
+ for (i = 0; i < NUM_BACKENDS; i++)
+ {
+ /* kind is singed char.
+ * We must check negative number.
+ */
+ int id = kind_list[i] + 128;
+
+ if (kind_list[i] == -1)
+ continue;
+
+ kind_map[id]++;
+ if (kind_map[id] > max_count)
+ {
+ max_kind = kind_list[i];
+ max_count = kind_map[id];
+ }
+ }
+
+ if (max_count != NUM_BACKENDS)
+ {
+ int trust_kind;
+
+ if (max_count <= NUM_BACKENDS / 2.0)
+ {
+ /* The group belonging to master is trusted. */
+ trust_kind = kind_list[MASTER_NODE_ID];
+ }
+ else /* max_count > NUM_BACKENDS / 2.0 */
+ {
+ trust_kind = max_kind;
+ }
+
+ for (i = 0; i < NUM_BACKENDS; i++)
+ {
+ if (kind_list[i] != -1 && trust_kind != kind_list[i])
{
- *kind = k;
+ /* degenerate */
+ pool_error("pool_process_query: %d th kind %c does not match with master connection kind %c",
+ i, kind_list[i], trust_kind);
+ degenerate_node[degenerate_node_num++] = i;
}
- else
- {
- if (k != *kind)
- {
- pool_error("pool_process_query: %d th kind %c does not match with master connection kind %c",
- i, k, *kind);
- pool_send_error_message(frontend, MAJOR(backend), "XX000",
- "kind mismatch between backends", "",
- "check data consistency between master and other db node", __FILE__, __LINE__);
+ }
+ }
- if (pool_config->replication_stop_on_mismatch)
- {
- notice_backend_error(i);
- child_exit(1);
- }
- else
- return POOL_ERROR;
- }
- }
+ if (degenerate_node_num)
+ {
+ pool_send_error_message(frontend, MAJOR(backend), "XX000",
+ "kind mismatch between backends", "",
+ "check data consistency between master and other db node",
+ __FILE__, __LINE__);
+ if (pool_config->replication_stop_on_mismatch)
+ {
+ degenerate_backend_set(degenerate_node, degenerate_node_num);
+ child_exit(1);
}
+ else
+ return POOL_ERROR;
}
+
+ *kind = kind_list[MASTER_NODE_ID];
return POOL_CONTINUE;
}
@@ -5088,7 +5132,7 @@ static int detect_error(POOL_CONNECTION *backend, char *error_code, int major, b
int is_error = 0;
char kind;
int readlen = 0, len;
- static char buf[1024]; /* memory space is large enough */
+ static char buf[8192]; /* memory space is large enough */
char *p, *str;
if (pool_read(backend, &kind, sizeof(kind)))
diff --git a/pool_stream.c b/pool_stream.c
index 1fd0d145c..a3715bf31 100644
--- a/pool_stream.c
+++ b/pool_stream.c
@@ -156,8 +156,8 @@ int pool_read(POOL_CONNECTION *cp, void *buf, int len)
if (cp->isbackend)
{
/* fatal error, notice to parent and exit */
- notice_backend_error(IS_MASTER_NODE_ID(cp->db_node_id));
- child_exit(1);
+ notice_backend_error(cp->db_node_id);
+ child_exit(1);
}
else
{
@@ -270,8 +270,8 @@ char *pool_read2(POOL_CONNECTION *cp, int len)
if (cp->isbackend)
{
/* fatal error, notice to parent and exit */
- notice_backend_error(IS_MASTER_NODE_ID(cp->db_node_id));
- child_exit(1);
+ notice_backend_error(cp->db_node_id);
+ child_exit(1);
}
else
{
@@ -453,7 +453,7 @@ int pool_flush(POOL_CONNECTION *cp)
{
if (cp->isbackend)
{
- notice_backend_error(IS_MASTER_NODE_ID(cp->db_node_id));
+ notice_backend_error(cp->db_node_id);
child_exit(1);
}
else
@@ -580,8 +580,8 @@ char *pool_read_string(POOL_CONNECTION *cp, int *len, int line)
if (cp->isbackend)
{
- notice_backend_error(IS_MASTER_NODE_ID(cp->db_node_id));
- child_exit(1);
+ notice_backend_error(cp->db_node_id);
+ child_exit(1);
}
else
{
[parent: 33130f82c10f]