pgpool2.git / summary / log / commit / refs
commit 8c22756510ead4ecdc0f560ef009d984409937bd
Author: Yugo Nagata <nagata@sraoss.co.jp>
Date: Tue Mar 19 07:29:46 2013 +0000
Fix to restart watchdog processes automatically when these exit abnormally.
main.c | 18 +++--
watchdog/watchdog.c | 188 +++++++++++++++++++++++++++++++++++++++++-------
watchdog/wd_child.c | 8 ++-
watchdog/wd_ext.h | 9 ++-
watchdog/wd_lifecheck.c | 10 ++-
watchdog/wd_udp.c | 23 ++++--
6 files changed, 211 insertions(+), 45 deletions(-)
diff --git a/main.c b/main.c
index 33e7b1d41..cda1aee6d 100644
--- a/main.c
+++ b/main.c
@@ -180,7 +180,6 @@ int my_proc_id;
static BackendStatusRecord backend_rec; /* Backend status record */
static pid_t worker_pid; /* pid of worker process */
-static pid_t watchdog_pid; /* pid of watchdog process */
BACKEND_STATUS* my_backend_status[MAX_NUM_BACKENDS]; /* Backend status buffer */
int my_master_node_id; /* Master node id buffer */
@@ -639,8 +638,7 @@ int main(int argc, char **argv)
/* start watchdog */
if (pool_config->use_watchdog )
{
- watchdog_pid = wd_main(1);
- if (watchdog_pid == 0)
+ if (!wd_main(1))
{
pool_error("wd_main error");
myexit(1);
@@ -1610,8 +1608,7 @@ static RETSIGTYPE exit_handler(int sig)
if (pool_config->use_watchdog)
{
- pool_log("watchdog_pid: %d", watchdog_pid);
- kill(watchdog_pid, sig);
+ wd_kill_watchdog(sig);
}
POOL_SETMASK(&UnBlockSig);
@@ -2316,6 +2313,17 @@ static void reaper(void)
pool_log("fork a new worker child pid %d", worker_pid);
}
+
+ /* exiting process was watchdog process */
+ else if (pool_config->use_watchdog && wd_is_watchdog_pid(pid))
+ {
+ if (!wd_reaper_watchdog(pid, status))
+ {
+ pool_error("wd_reaper failed");
+ myexit(1);
+ }
+ }
+
else
{
if (WIFSIGNALED(status))
diff --git a/watchdog/watchdog.c b/watchdog/watchdog.c
index 1188d3562..4fa820fc4 100644
--- a/watchdog/watchdog.c
+++ b/watchdog/watchdog.c
@@ -40,34 +40,25 @@
WdInfo * WD_List = NULL; /* watchdog server list */
unsigned char * WD_Node_List = NULL; /* node list */
+
pid_t wd_ppid = 0;
+static pid_t lifecheck_pid;
static pid_t child_pid;
-
static pid_t reader_pid[WD_MAX_IF_NUM];
static pid_t writer_pid[WD_MAX_IF_NUM];
pid_t wd_main(int fork_wait_time);
-static void child_wait(int signo);
+void wd_kill_watchdog(int sig);
+int wd_reaper_watchdog(pid_t pid, int status);
+static pid_t fork_a_lifecheck(int fork_wait_time);
static void wd_exit(int exit_status);
static int wd_check_config(void);
static int has_sticky_bit(char * path);
-static void
-child_wait(int signo)
-{
- pid_t pid = 0;
-
- do {
- int ret;
- pid = waitpid(-1,&ret,WNOHANG);
- } while(pid > 0);
-}
-
static void
wd_exit(int exit_signo)
{
sigset_t mask;
- int i;
sigemptyset(&mask);
sigaddset(&mask, SIGTERM);
@@ -78,16 +69,22 @@ wd_exit(int exit_signo)
wd_notice_server_down();
- kill (child_pid, exit_signo);
+ exit(0);
+}
+
+void
+wd_kill_watchdog(int sig)
+{
+ int i;
+
+ kill (lifecheck_pid, sig);
+ kill (child_pid, sig);
for (i = 0; i < pool_config->other_wd->num_udp_if; i++)
{
- kill (reader_pid[i], exit_signo);
- kill (writer_pid[i], exit_signo);
+ kill (reader_pid[i], sig);
+ kill (writer_pid[i], sig);
}
- child_wait(0);
-
- exit(0);
}
static int
@@ -113,20 +110,19 @@ wd_check_config(void)
}
return status;
-
}
pid_t
wd_main(int fork_wait_time)
{
int status = WD_INIT;
- pid_t pid = 0;
int i;
if (!pool_config->use_watchdog)
{
return 0;
}
+
/* check pool_config data */
status = wd_check_config();
if (status != WD_OK)
@@ -150,15 +146,14 @@ wd_main(int fork_wait_time)
if (child_pid < 0 )
{
pool_error("launch wd_child failed");
- return child_pid;
+ return 0;
}
-
if (!strcmp(pool_config->watchdog_mode, "udp"))
{
for (i = 0; i < pool_config->other_wd->num_udp_if; i++)
{
- /* reader */
+ /* reader process */
reader_pid[i] = wd_reader(1, pool_config->other_wd->udp_if[i]);
if (reader_pid[i] < 0 )
{
@@ -166,7 +161,7 @@ wd_main(int fork_wait_time)
return reader_pid[i];
}
- /* writer */
+ /* writer process */
writer_pid[i] = wd_writer(1, pool_config->other_wd->udp_if[i]);
if (writer_pid[i] < 0 )
{
@@ -177,18 +172,40 @@ wd_main(int fork_wait_time)
}
/* fork lifecheck process*/
+ lifecheck_pid = fork_a_lifecheck(fork_wait_time);
+ if (lifecheck_pid < 0 )
+ {
+ pool_error("launch lifecheck process failed");
+ return 0;
+ }
+
+ return lifecheck_pid;
+}
+
+
+/* fork lifecheck process*/
+static pid_t
+fork_a_lifecheck(int fork_wait_time)
+{
+ pid_t pid;
+
pid = fork();
if (pid != 0)
{
+ if (pid == -1)
+ pool_error("fork_a_lifecheck: fork() failed.");
+
return pid;
}
if (fork_wait_time > 0) {
sleep(fork_wait_time);
}
-
+
myargv = save_ps_display_args(myargc, myargv);
+ POOL_SETMASK(&UnBlockSig);
+
init_ps_display("", "", "", "");
signal(SIGTERM, wd_exit);
@@ -215,9 +232,126 @@ wd_main(int fork_wait_time)
wd_lifecheck();
sleep(pool_config->wd_interval);
}
+
return pid;
}
+int
+wd_is_watchdog_pid(pid_t pid)
+{
+ int i;
+
+ if (pid == lifecheck_pid || pid == child_pid)
+ {
+ return 1;
+ }
+
+ for (i = 0; i < pool_config->other_wd->num_udp_if; i++)
+ {
+ if (pid == reader_pid[i] || pid == writer_pid[i])
+ {
+ return 1;
+ }
+ }
+
+ return 0;
+}
+
+int
+wd_reaper_watchdog(pid_t pid, int status)
+{
+ int i;
+
+ /* exiting process was watchdog lifecheck process */
+ if (pid == lifecheck_pid)
+ {
+ if (WIFSIGNALED(status))
+ pool_debug("watchdog lifecheck process %d exits with status %d by signal %d",
+ pid, status, WTERMSIG(status));
+ else
+ pool_debug("watchdog lifecheck process %d exits with status %d", pid, status);
+
+ lifecheck_pid = fork_a_lifecheck(1);
+
+ if (lifecheck_pid < 0)
+ {
+ pool_error("wd_reaper: fork a watchdog lifecheck process failed");
+ return 0;
+ }
+
+ pool_log("fork a new watchdog lifecheck pid %d", lifecheck_pid);
+ }
+
+ /* exiting process was watchdog child process */
+ else if (pid == child_pid)
+ {
+ if (WIFSIGNALED(status))
+ pool_debug("watchdog child process %d exits with status %d by signal %d",
+ pid, status, WTERMSIG(status));
+ else
+ pool_debug("watchdog child process %d exits with status %d", pid, status);
+
+ child_pid = wd_child(1);
+
+ if (child_pid < 0)
+ {
+ pool_error("wd_reaper: fork a watchdog child process failed");
+ return 0;
+ }
+
+ pool_log("fork a new watchdog child pid %d", child_pid);
+ }
+
+ /* exiting process was reader/writer process */
+ else
+ {
+ for (i = 0; i < pool_config->other_wd->num_udp_if; i++)
+ {
+ if (pid == reader_pid[i])
+ {
+ if (WIFSIGNALED(status))
+ pool_debug("watchdog reader process %d exits with status %d by signal %d",
+ pid, status, WTERMSIG(status));
+ else
+ pool_debug("watchdog reader process %d exits with status %d", pid, status);
+
+ reader_pid[i] = wd_reader(1, pool_config->other_wd->udp_if[i]);
+
+ if (reader_pid[i] < 0)
+ {
+ pool_error("wd_reaper: fork a watchdog reader process failed");
+ return 0;
+ }
+
+ pool_log("fork a new watchdog reader pid %d", reader_pid[i]);
+ break;
+ }
+
+ else if (pid == writer_pid[i])
+ {
+ if (WIFSIGNALED(status))
+ pool_debug("watchdog writer process %d exits with status %d by signal %d",
+ pid, status, WTERMSIG(status));
+ else
+ pool_debug("watchdog writer process %d exits with status %d", pid, status);
+
+ writer_pid[i] = wd_writer(1, pool_config->other_wd->udp_if[i]);
+
+ if (writer_pid[i] < 0)
+ {
+ pool_error("wd_reaper: fork a watchdog writer process failed");
+ return 0;
+ }
+
+ pool_log("fork a new watchdog writer pid %d", writer_pid[i]);
+ break;
+ }
+ }
+ }
+
+ return 1;
+}
+
int
wd_chk_sticky(void)
{
diff --git a/watchdog/wd_child.c b/watchdog/wd_child.c
index f46c4c3af..6f3d0d131 100644
--- a/watchdog/wd_child.c
+++ b/watchdog/wd_child.c
@@ -49,6 +49,9 @@ wd_child(int fork_wait_time)
pid = fork();
if (pid != 0)
{
+ if (pid == -1)
+ pool_error("wd_child: fork() failed.");
+
return pid;
}
@@ -59,6 +62,8 @@ wd_child(int fork_wait_time)
myargv = save_ps_display_args(myargc, myargv);
+ POOL_SETMASK(&UnBlockSig);
+
signal(SIGTERM, wd_child_exit);
signal(SIGINT, wd_child_exit);
signal(SIGQUIT, wd_child_exit);
@@ -68,9 +73,6 @@ wd_child(int fork_wait_time)
signal(SIGUSR2, SIG_IGN);
signal(SIGPIPE, SIG_IGN);
signal(SIGALRM, SIG_IGN);
- //TODO
- // reload
- // wakeup, failover??
init_ps_display("", "", "", "");
diff --git a/watchdog/wd_ext.h b/watchdog/wd_ext.h
index 2b4d987e5..33965fa40 100644
--- a/watchdog/wd_ext.h
+++ b/watchdog/wd_ext.h
@@ -30,6 +30,9 @@
extern pid_t wd_ppid;
extern pid_t wd_main(int fork_wait_time);
extern int wd_chk_sticky(void);
+extern int wd_is_watchdog_pid(pid_t pid);
+extern int wd_reaper_watchdog(pid_t pid, int status);
+extern void wd_kill_eatchdog(int sig);
/* wd_child.c */
extern pid_t wd_child(int fork_wait_time);
@@ -39,8 +42,7 @@ extern int wd_init(void);
/* wd_list.c */
extern int wd_set_wd_list(char * hostname, int pgpool_port, int wd_port, char * delegate_ip, struct timeval * tv, int status);
-extern int wd_add_wd_list(WdDesc * other_wd);
-extern int wd_set_wd_info(WdInfo * info);
+extern int wd_add_wd_list(WdDesc * other_wd); extern int wd_set_wd_info(WdInfo * info);
extern WdInfo * wd_is_exist_master(void);
extern int wd_am_I_oldest(void);
extern int wd_set_myself(struct timeval * tv, int status);
@@ -63,6 +65,7 @@ extern int wd_send_failback_request(int node_id);
extern int wd_degenerate_backend_set(int *node_id_set, int count);
extern int wd_promote_backend(int node_id);
extern int wd_set_node_mask (WD_PACKET_NO packet_no, int *node_id_set, int count);
+
/* wd_ping.c */
extern int wd_is_upper_ok(char * server_list);
extern int wd_is_unused_ip(char * ip);
@@ -77,7 +80,7 @@ extern int is_wd_lifecheck_ready(void);
extern int wd_lifecheck(void);
extern int wd_ping_pgpool(WdInfo * pgpool);
-///hogehoge
+/* wd_udp.c */
extern int wd_create_udp_send_socket(WdUdpIf udp_if);
extern int wd_create_udp_recv_socket(WdUdpIf udp_if);
extern int wd_udp_write(int sock, WdUdpPacket * pkt, int len, const char * destination);
diff --git a/watchdog/wd_lifecheck.c b/watchdog/wd_lifecheck.c
index 5de18e72a..b38734b98 100644
--- a/watchdog/wd_lifecheck.c
+++ b/watchdog/wd_lifecheck.c
@@ -193,7 +193,7 @@ check_pgpool_status_by_udp(void)
}
else
{
- pool_debug("check_pgpool_status_by_udp: OK");
+ pool_debug("check_pgpool_status_by_udp: OK; status %d", p->status);
}
}
else
@@ -202,14 +202,17 @@ check_pgpool_status_by_udp(void)
if (interval > pool_config->wd_udp_deadtime)
{
- pool_debug("check_pgpool_status_by_udp: NG; the latest heartbeat received %d seconds ago", interval);
+ pool_debug("check_pgpool_status_by_udp: the latest heartbeat received %d seconds ago", interval);
+ pool_debug("check_pgpool_status_by_udp: NG; status %d", p->status);
+
pool_log("check_pgpool_status_by_udp: lifecheck failed. pgpool %d (%s:%d) seems not to be working",
cnt, p->hostname, p->pgpool_port);
pgpool_down(p);
}
else
{
- pool_debug("check_pgpool_status_by_udp: OK; the latest heartbeat received %d secconds ago", interval);
+ pool_debug("check_pgpool_status_by_udp: the latest heartbeat received %d secconds ago", interval);
+ pool_debug("check_pgpool_status_by_udp: OK; status %d", p->status);
}
}
@@ -372,6 +375,7 @@ static int
pgpool_down(WdInfo * pool)
{
int rtn = WD_DOWN;
+
if ((WD_List->status == WD_NORMAL) &&
(pool->status == WD_MASTER))
{
diff --git a/watchdog/wd_udp.c b/watchdog/wd_udp.c
index a2c198282..c69fdd312 100644
--- a/watchdog/wd_udp.c
+++ b/watchdog/wd_udp.c
@@ -161,7 +161,7 @@ wd_create_udp_recv_socket(WdUdpIf udp_if)
close(sock);
return -1;
}
- pool_log("wd_create_udp_recv_socket: bind send socket to device: %s", i.ifr_name);
+ pool_log("wd_create_udp_recv_socket: bind receive socket to device: %s", i.ifr_name);
}
#endif
@@ -217,6 +217,12 @@ wd_udp_write(int sock, WdUdpPacket * pkt, int len, const char * host)
struct hostent *hp;
WdUdpPacket buf;
+ if (!host || !strlen(host))
+ {
+ pool_error("wd_udp_write: host name is empty");
+ return -1;
+ }
+
hp = gethostbyname(host);
if ((hp == NULL) || (hp->h_addrtype != AF_INET))
{
@@ -237,7 +243,7 @@ wd_udp_write(int sock, WdUdpPacket * pkt, int len, const char * host)
pool_error("wd_udp_write: failed to sent packet to %s", host);
return WD_NG;
}
- pool_debug("wd_udp_write: send %d byte paeckt", rtn);
+ pool_debug("wd_udp_write: send %d byte packet", rtn);
return WD_OK;
}
@@ -266,7 +272,7 @@ wd_udp_read(int sock, WdUdpPacket * pkt)
}
else
{
- pool_debug("wd_udp_read: received %d byte paeckt", rtn);
+ pool_debug("wd_udp_read: received %d byte packet", rtn);
}
ntoh_wd_udp_packet(pkt, &buf);
@@ -288,6 +294,9 @@ pid_t wd_reader(int fork_wait_time, WdUdpIf udp_if)
pid = fork();
if (pid != 0)
{
+ if (pid == -1)
+ pool_error("wd_reader: fork() failed.");
+
return pid;
}
@@ -298,6 +307,8 @@ pid_t wd_reader(int fork_wait_time, WdUdpIf udp_if)
myargv = save_ps_display_args(myargc, myargv);
+ POOL_SETMASK(&UnBlockSig);
+
signal(SIGTERM, reader_exit);
signal(SIGINT, reader_exit);
signal(SIGQUIT, reader_exit);
@@ -374,6 +385,9 @@ pid_t wd_writer(int fork_wait_time, WdUdpIf udp_if)
pid = fork();
if (pid != 0)
{
+ if (pid == -1)
+ pool_error("wd_writer: fork() failed.");
+
return pid;
}
@@ -384,6 +398,8 @@ pid_t wd_writer(int fork_wait_time, WdUdpIf udp_if)
myargv = save_ps_display_args(myargc, myargv);
+ POOL_SETMASK(&UnBlockSig);
+
signal(SIGTERM, writer_exit);
signal(SIGINT, writer_exit);
signal(SIGQUIT, writer_exit);
@@ -396,7 +412,6 @@ pid_t wd_writer(int fork_wait_time, WdUdpIf udp_if)
init_ps_display("", "", "", "");
-
if ( (sock = wd_create_udp_send_socket(udp_if)) < 0)
{
pool_error("wd_writer: socket create failed");
[parent: a9db38b33971]