pgpool2.git / summary / log / commit / refs
commit dcfe7810e158ad1a48617d07a466e686dd0ba04c
Author: Yugo Nagata <nagata@sraoss.co.jp>
Date: Tue Jul 30 06:32:41 2013 +0000
Fix a hang that occurs when mutiple pgpools startup before all backend startup
Specify a timeout value for lock-waiting loops.
main.c | 14 ++------
watchdog/wd_ext.h | 5 +--
watchdog/wd_interlock.c | 94 ++++++++++++++++++++++++++++++++++++++++++-------
watchdog/wd_list.c | 31 +++++++++++++---
watchdog/wd_packet.c | 3 +-
5 files changed, 115 insertions(+), 32 deletions(-)
diff --git a/main.c b/main.c
index 160aaa12f..f0af522ad 100644
--- a/main.c
+++ b/main.c
@@ -1697,6 +1697,7 @@ static void failover(void)
{
int i;
int node_id;
+ bool by_health_check;
int new_master;
int new_primary;
int nodes[MAX_NUM_BACKENDS];
@@ -1761,17 +1762,8 @@ static void failover(void)
/* start of command inter-lock with watchdog */
if (pool_config->use_watchdog)
{
- wd_start_interlock();
-
- /*
- * if it is due to DB down detection by healthcheck, send failover request
- * to other pgpools because detection of DB down on the others may be late.
- */
- if (wd_am_I_lock_holder() &&
- !failover_request && Req_info->kind == NODE_DOWN_REQUEST)
- {
- wd_degenerate_backend_set(&node_id, 1);
- }
+ by_health_check = (!failover_request && Req_info->kind==NODE_DOWN_REQUEST);
+ wd_start_interlock(by_health_check);
}
/* failback request? */
diff --git a/watchdog/wd_ext.h b/watchdog/wd_ext.h
index 8f0e513c9..5f47dcab4 100644
--- a/watchdog/wd_ext.h
+++ b/watchdog/wd_ext.h
@@ -51,11 +51,12 @@ extern WdInfo * wd_is_alive_master(void);
extern WdInfo * wd_get_lock_holder(void);
extern WdInfo * wd_get_interlocking(void);
-extern bool wd_is_interlocking_all(void);
+extern bool wd_are_interlocking_all(void);
extern void wd_set_lock_holder(WdInfo *p, bool value);
extern void wd_set_interlocking(WdInfo *info, bool value);
extern void wd_clear_interlocking_info(void);
extern bool wd_is_contactable_master(void);
+extern bool wd_are_contactable_all(void);
extern WdInfo * wd_get_watchdog_info(int num);
/* wd_packet.c */
@@ -107,7 +108,7 @@ extern pid_t wd_hb_sender(int fork_wait_time, WdHbIf hb_if);
/* wd_interlock.c */
extern int wd_init_interlock(void);
-extern void wd_start_interlock(void);
+extern void wd_start_interlock(bool by_health_check);
extern void wd_end_interlock(void);
extern void wd_leave_interlock(void);
extern void wd_wait_for_lock(WD_LOCK_ID lock_id);
diff --git a/watchdog/wd_interlock.c b/watchdog/wd_interlock.c
index 7fc70520b..c64b4c62b 100644
--- a/watchdog/wd_interlock.c
+++ b/watchdog/wd_interlock.c
@@ -34,7 +34,7 @@
static volatile bool * WD_Locks;
int wd_init_interlock(void);
-void wd_start_interlock(void);
+void wd_start_interlock(bool by_health_check);
void wd_end_interlock(void);
void wd_leave_interlock(void);
void wd_wait_for_lock(WD_LOCK_ID lock_id);
@@ -47,8 +47,14 @@ static void wd_update_lock_holder(void);
static int wd_assume_lock_holder(void);
static void wd_resign_lock_holder(void);
static void wd_lock_all(void);
+static void wd_confirm_contactable(void);
+
static void sleep_in_waiting(void);
+#define WD_INTERLOCK_WAIT_MSEC 500
+#define WD_INTERLOCK_TIMEOUT_SEC 10
+#define WD_INTERLOCK_WAIT_COUNT ((int) ((WD_INTERLOCK_TIMEOUT_SEC * 1000)/WD_INTERLOCK_WAIT_MSEC))
+
/* initialize interlock */
int
wd_init_interlock(void)
@@ -69,10 +75,16 @@ wd_init_interlock(void)
}
/* notify to start interlocking */
-void wd_start_interlock(void)
+void wd_start_interlock(bool by_health_check)
{
+ int count;
+ int node_id;
+
pool_log("wd_start_interlock: start interlocking");
+ /* confirm other pgpools are contactable */
+ wd_confirm_contactable();
+
/* lock all the resource */
wd_lock_all();
@@ -82,34 +94,52 @@ void wd_start_interlock(void)
/* try to assume lock holder */
wd_assume_lock_holder();
+ /*
+ * If it is due to DB down detection by healthcheck, send failover request
+ * to other pgpools because detection of DB down on the others may be late.
+ */
+ if (by_health_check && wd_am_I_lock_holder())
+ {
+ node_id = Req_info->node_id[0];
+ wd_degenerate_backend_set(&node_id, 1);
+ }
+
/* wait for all pgpools starting interlock */
+ count = WD_INTERLOCK_WAIT_COUNT;
while (wd_is_locked(WD_FAILOVER_START_LOCK))
{
if (WD_MYSELF->status == WD_DOWN)
{
wd_set_lock_holder(WD_MYSELF, false);
- break;
+ return;
}
- if (wd_am_I_lock_holder() && wd_is_interlocking_all())
+ if (wd_am_I_lock_holder() && wd_are_interlocking_all())
{
wd_unlock(WD_FAILOVER_START_LOCK);
}
sleep_in_waiting();
+ if (--count < 0)
+ {
+ pool_error("wd_start_interlock: timed out");
+ break;
+ }
}
-
}
/* notify to end interlocking */
void wd_end_interlock(void)
{
+ int count;
+
pool_log("wd_end_interlock: end interlocking");
wd_set_interlocking(WD_MYSELF, false);
wd_send_packet_no(WD_END_INTERLOCK);
/* wait for all pgpools ending interlock */
+ count = WD_INTERLOCK_WAIT_COUNT;
while (wd_is_locked(WD_FAILOVER_END_LOCK))
{
if (WD_MYSELF->status == WD_DOWN)
@@ -124,8 +154,16 @@ void wd_end_interlock(void)
}
sleep_in_waiting();
+ if (--count < 0)
+ {
+ pool_error("wd_end_interlock: timed out");
+ break;
+ }
}
+ if (wd_am_I_lock_holder())
+ wd_resign_lock_holder();
+
wd_clear_interlocking_info();
}
@@ -150,18 +188,26 @@ bool wd_am_I_lock_holder(void)
/* waiting for the lock or to be lock holder */
void wd_wait_for_lock(WD_LOCK_ID lock_id)
{
+ int count;
+
+ count = WD_INTERLOCK_WAIT_COUNT;
while (wd_is_locked(lock_id))
{
if (WD_MYSELF->status == WD_DOWN)
{
wd_set_lock_holder(WD_MYSELF, false);
- break;
+ return;
}
if (wd_am_I_lock_holder())
break;
sleep_in_waiting();
+ if (--count < 0)
+ {
+ pool_error("wd_wait_for_lock: timed out");
+ break;
+ }
}
}
@@ -180,14 +226,15 @@ wd_assume_lock_holder(void)
return WD_NG;
/*
- * confirm contatable master exists,
- * otherwise WD_STAND_FOR_LOCK_HOLDER always returns WD_OK,
- * eventually multiple pgpools become lock_holder
+ * confirm contatable master exists;
+ * If contactable master doesn't exists, WD_STAND_FOR_LOCK_HOLDER always
+ * returns WD_OK, eventually multiple pgpools can become lock_holder.
+ * Down of the host on which master pgpool was working is the case.
*/
while (!wd_is_contactable_master())
{
if (WD_MYSELF->status == WD_DOWN)
- break;
+ return WD_NG;
}
/* I'm master and not lock holder, or I succeeded to become lock holder */
@@ -207,7 +254,7 @@ wd_assume_lock_holder(void)
/*
* update information of who's lock holder
- * Note that the lock holder pgpool can go down accidentally.
+ * Note that a lock holder pgpool may go down accidentally during interlocking.
*/
static void
wd_update_lock_holder(void)
@@ -216,7 +263,7 @@ wd_update_lock_holder(void)
while (wd_get_lock_holder() == NULL)
{
if (WD_MYSELF->status == WD_DOWN)
- break;
+ return;
wd_assume_lock_holder();
}
@@ -267,9 +314,30 @@ wd_unlock(WD_LOCK_ID lock_id)
return rtn;
}
+/* confirm other pgpools are contactable */
+static void
+wd_confirm_contactable(void)
+{
+ int count;
+
+ count = WD_INTERLOCK_WAIT_COUNT;
+ while (!wd_are_contactable_all())
+ {
+ if (WD_MYSELF->status == WD_DOWN)
+ return;
+
+ sleep_in_waiting();
+ if (--count < 0)
+ {
+ pool_error("wd_confirm_contactable: timed out");
+ break;
+ }
+ }
+}
+
static void
sleep_in_waiting(void)
{
- struct timeval t = {0, 100000};
+ struct timeval t = {0, WD_INTERLOCK_WAIT_MSEC * 1000};
select(0, NULL, NULL, NULL, &t);
}
diff --git a/watchdog/wd_list.c b/watchdog/wd_list.c
index 83ec4e927..ae0938703 100644
--- a/watchdog/wd_list.c
+++ b/watchdog/wd_list.c
@@ -36,7 +36,7 @@ int wd_set_wd_info(WdInfo * info);
WdInfo * wd_is_exist_master(void);
WdInfo * wd_get_lock_holder(void);
WdInfo * wd_get_interlocking(void);
-bool wd_is_interlocking_all(void);
+bool wd_are_interlocking_all(void);
void wd_set_lock_holder(WdInfo *info, bool value);
void wd_set_interlocking(WdInfo *info, bool value);
void wd_clear_interlocking_info(void);
@@ -44,6 +44,7 @@ int wd_am_I_oldest(void);
int wd_set_myself(struct timeval * tv, int status);
WdInfo * wd_is_alive_master(void);
bool wd_is_contactable_master(void);
+bool wd_are_contactable_all(void);
/* add or modify watchdog information list */
int
@@ -260,7 +261,7 @@ wd_get_interlocking(void)
/* if all pgpool are in interlocking return true, otherwise false */
bool
-wd_is_interlocking_all(void)
+wd_are_interlocking_all(void)
{
WdInfo * p = WD_List;
bool rtn = true;
@@ -357,8 +358,7 @@ wd_is_alive_master(void)
master = wd_is_exist_master();
if (master != NULL)
{
- if ((!strcmp(pool_config->wd_lifecheck_method, MODE_HEARTBEAT)
- && wd_check_heartbeat(master) == WD_OK) ||
+ if ((!strcmp(pool_config->wd_lifecheck_method, MODE_HEARTBEAT)) ||
(!strcmp(pool_config->wd_lifecheck_method, MODE_QUERY)
&& wd_ping_pgpool(master) == WD_OK))
{
@@ -386,7 +386,7 @@ wd_is_contactable_master(void)
/* for updating contactable flag */
wd_update_info();
- master = wd_is_alive_master();
+ master = wd_is_exist_master();
if (master != NULL)
{
return master->is_contactable;
@@ -395,6 +395,27 @@ wd_is_contactable_master(void)
return false;
}
+bool
+wd_are_contactable_all(void)
+{
+ WdInfo * p = WD_List;
+
+ /* for updating contactable flag */
+ wd_update_info();
+
+ while (p->status != WD_END)
+ {
+ if ((p->status == WD_NORMAL || p->status == WD_MASTER) &&
+ !p->is_contactable)
+ {
+ return false;
+ }
+ p++;
+ }
+
+ return true;
+}
+
/*
* get watchdog information of specified index
*/
diff --git a/watchdog/wd_packet.c b/watchdog/wd_packet.c
index 78ce5eee1..04f11e6fb 100644
--- a/watchdog/wd_packet.c
+++ b/watchdog/wd_packet.c
@@ -654,7 +654,8 @@ send_packet_4_nodes(WdPacket *packet, WD_SEND_TYPE type)
while (p->status != WD_END)
{
/* don't send packet to pgpool in down */
- if (p->status == WD_DOWN)
+ if (p->status == WD_DOWN ||
+ (packet->packet_no != WD_ADD_REQ && p->status == WD_INIT))
{
p->is_contactable = false;
p++;
[parent: 2840a90d731d]