pgpool2.git / summary / log / commit / refs
commit e55c230d674ea221c09e8266a68d119f1a3a2dac
Author: Muhammad Usama <m.usama@gmail.com>
Date: Mon Mar 07 15:33:00 2016 +0000
Fix for [pgpool-general: 4519] Worker Processes Exit and Are Not Re-spawned
The problem was due to a logical mistake in the code for checking the exiting
child process type when the watchdog is enabled.
I have also changed the severity of the message from FATAL to LOG, emitted
for child exits due to max connection reached.
src/include/utils/elog.h | 9 ++++++---
src/main/pgpool_main.c | 16 +++++++++++++---
src/pcp_con/pcp_child.c | 2 +-
src/protocol/child.c | 26 +++++++++++++-------------
src/protocol/pool_connection_pool.c | 2 +-
src/protocol/pool_proto_modules.c | 2 +-
src/utils/error/elog.c | 2 +-
src/utils/pool_ssl.c | 4 ++--
src/utils/pool_stream.c | 14 +++++++-------
src/watchdog/watchdog.c | 2 +-
src/watchdog/wd_lifecheck.c | 2 +-
11 files changed, 47 insertions(+), 34 deletions(-)
diff --git a/src/include/utils/elog.h b/src/include/utils/elog.h
index 163f39522..bfef1106d 100644
--- a/src/include/utils/elog.h
+++ b/src/include/utils/elog.h
@@ -124,9 +124,12 @@ typedef enum
#define FRONTEND_ERROR 23 /* transformed to ERROR at errstart */
/* #define DEBUG DEBUG1 */ /* Backward compatibility with pre-7.3 */
-#define POOL_EXIT_SUCCESS 0 /* failure exit and child gets restarted*/
-#define POOL_EXIT_NOFATAL 1 /* failure exit and child gets restarted*/
-#define POOL_EXIT_FATAL 3 /* This exit code from child takes down the pgpool main with it */
+#define POOL_EXIT_NO_RESTART 0 /* child exiting with this error will not be
+ * restarted by pgpool main process
+ * but the pgpool main will remain unaffected by this */
+#define POOL_EXIT_AND_RESTART 1 /* child process exit with this error code will
+ * be restarted by the pgpool main */
+#define POOL_EXIT_FATAL 3 /* This exit code from child also takes down the pgpool main with it */
diff --git a/src/main/pgpool_main.c b/src/main/pgpool_main.c
index 2691d87e4..2e9351806 100644
--- a/src/main/pgpool_main.c
+++ b/src/main/pgpool_main.c
@@ -2124,7 +2124,7 @@ static char* process_name_from_pid(pid_t pid)
* terminating child but shutdowns the pgpool-II. This allow
* the child process to inform parent process of fatal failures which needs
* to be rectified (e.g startup failure) by user for smooth running of system.
- * Also the child exits with success status POOL_EXIT_SUCCESS does not gets
+ * Also the child exits with success status POOL_EXIT_NO_RESTART does not gets
* restarted.
*/
static void reaper(void)
@@ -2158,6 +2158,7 @@ static void reaper(void)
pid_t new_pid = 0;
bool shutdown_system = false;
bool restart_child = true;
+ bool found = false;
char *exiting_process_name = process_name_from_pid(pid);
/*
@@ -2171,7 +2172,7 @@ static void reaper(void)
(errmsg("%s process with pid: %d exit with FATAL ERROR. pgpool-II will be shutdown",exiting_process_name, pid)));
shutdown_system = true;
}
- else if(WEXITSTATUS(status) == POOL_EXIT_SUCCESS)
+ else if(WEXITSTATUS(status) == POOL_EXIT_NO_RESTART)
{
ereport(DEBUG1,
(errmsg("%s process with pid: %d exit with SUCCESS. child will not be restarted",exiting_process_name, pid)));
@@ -2196,6 +2197,7 @@ static void reaper(void)
/* if exiting child process was PCP handler */
if (pid == pcp_pid)
{
+ found = true;
if(restart_child)
{
pcp_pid = pcp_fork_a_child(pcp_unix_fd, pcp_inet_fd, pcp_conf_file);
@@ -2208,6 +2210,7 @@ static void reaper(void)
/* exiting process was worker process */
else if (pid == worker_pid)
{
+ found = true;
if(restart_child)
{
worker_pid = worker_fork_a_child();
@@ -2222,6 +2225,7 @@ static void reaper(void)
{
if (watchdog_pid == pid)
{
+ found = true;
if(restart_child)
{
watchdog_pid = initialize_watchdog();
@@ -2232,6 +2236,7 @@ static void reaper(void)
}
else if (wd_lifecheck_pid == pid)
{
+ found = true;
if(restart_child)
{
wd_lifecheck_pid = initialize_watchdog_lifecheck();
@@ -2241,13 +2246,17 @@ static void reaper(void)
wd_lifecheck_pid = 0;
}
}
- else
+ /* we are not able to identify the exiting process yet.
+ * check if the exiting process was child process (handling PG clients)
+ */
+ if (found == false)
{
/* look for exiting child's pid */
for (i=0;i<pool_config->num_init_children;i++)
{
if (pid == process_info[i].pid)
{
+ found = true;
/* if found, fork a new child */
if (!switching && !exiting && restart_child)
{
@@ -2264,6 +2273,7 @@ static void reaper(void)
if(shutdown_system)
ereport(FATAL,
(errmsg("%s process exit with fatal error. exiting pgpool-II",exiting_process_name)));
+
else if(restart_child && new_pid)
{
/* Report if the child was restarted */
diff --git a/src/pcp_con/pcp_child.c b/src/pcp_con/pcp_child.c
index 13a2d1225..ded032fb1 100644
--- a/src/pcp_con/pcp_child.c
+++ b/src/pcp_con/pcp_child.c
@@ -322,7 +322,7 @@ static void reaper(void)
if(WEXITSTATUS(status) == POOL_EXIT_FATAL)
ereport(LOG,
(errmsg("PCP worker process with pid: %d exit with FATAL ERROR.", pid)));
- else if(WEXITSTATUS(status) == POOL_EXIT_SUCCESS)
+ else
ereport(LOG,
(errmsg("PCP process with pid: %d exit with SUCCESS.", pid)));
}
diff --git a/src/protocol/child.c b/src/protocol/child.c
index e0a0f4cf1..ab3bf39cf 100644
--- a/src/protocol/child.c
+++ b/src/protocol/child.c
@@ -192,7 +192,7 @@ void do_child(int *fds)
/* initialize connection pool */
if (pool_init_cp())
{
- child_exit(1);
+ child_exit(POOL_EXIT_AND_RESTART);
}
/*
@@ -253,7 +253,7 @@ void do_child(int *fds)
{
ereport(LOG,
(errmsg("child exiting, %d connections reached", pool_config->child_max_connections)));
- child_exit(2);
+ child_exit(POOL_EXIT_AND_RESTART);
}
}
@@ -299,7 +299,7 @@ void do_child(int *fds)
{
ereport(DEBUG1,
(errmsg("child life %d seconds expired", pool_config->child_life_time)));
- child_exit(2);
+ child_exit(POOL_EXIT_AND_RESTART);
}
continue;
}
@@ -395,12 +395,12 @@ void do_child(int *fds)
if ( ( pool_config->child_max_connections > 0 ) &&
( connections_count >= pool_config->child_max_connections ) )
{
- ereport(FATAL,
- (return_code(2),
- errmsg("child exiting, %d connections reached", pool_config->child_max_connections)));
+ ereport(LOG,
+ (errmsg("child exiting, %d connections reached", pool_config->child_max_connections)));
+ child_exit(POOL_EXIT_AND_RESTART);
}
}
- child_exit(0);
+ child_exit(POOL_EXIT_NO_RESTART);
}
/* -------------------------------------------------------------------
@@ -933,7 +933,7 @@ static RETSIGTYPE die(int sig)
case SIGINT: /* fast shutdown */
case SIGQUIT: /* immediate shutdown */
- child_exit(0);
+ child_exit(POOL_EXIT_NO_RESTART);
break;
default:
ereport(LOG,
@@ -1005,7 +1005,7 @@ static RETSIGTYPE authentication_timeout(int sig)
ereport(LOG,
(errmsg("authentication timeout")));
- child_exit(1);
+ child_exit(POOL_EXIT_AND_RESTART);
}
static void enable_authentication_timeout(void)
@@ -1742,7 +1742,7 @@ void check_stop_request(void)
if (exit_request)
{
reset_variables();
- child_exit(0);
+ child_exit(POOL_EXIT_NO_RESTART);
}
}
@@ -1781,7 +1781,7 @@ static void check_restart_request(void)
errdetail("restarting myself")));
pool_get_my_process_info()->need_to_restart = 0;
- child_exit(1);
+ child_exit(POOL_EXIT_AND_RESTART);
}
}
@@ -2055,8 +2055,8 @@ static void validate_backend_connectivity(int front_end_fd)
{
if ((cp = pool_open(front_end_fd,false)) == NULL)
{
- close(front_end_fd); //todo
- child_exit(1);
+ close(front_end_fd);
+ child_exit(POOL_EXIT_AND_RESTART);
}
sp = read_startup_packet(cp);
ereport(DEBUG1,
diff --git a/src/protocol/pool_connection_pool.c b/src/protocol/pool_connection_pool.c
index f9eb800bf..a7de1b16a 100644
--- a/src/protocol/pool_connection_pool.c
+++ b/src/protocol/pool_connection_pool.c
@@ -858,7 +858,7 @@ static POOL_CONNECTION_POOL *new_connection(POOL_CONNECTION_POOL *p)
(errmsg("failed to create a backend connection"),
errdetail("not executing failover because fail_over_on_backend_error is off")));
}
- child_exit(1);
+ child_exit(POOL_EXIT_AND_RESTART);
}
p->info[i].create_time = time(NULL);
diff --git a/src/protocol/pool_proto_modules.c b/src/protocol/pool_proto_modules.c
index 168a7eaf3..5e871233e 100644
--- a/src/protocol/pool_proto_modules.c
+++ b/src/protocol/pool_proto_modules.c
@@ -1490,7 +1490,7 @@ POOL_STATUS ReadyForQuery(POOL_CONNECTION *frontend,
free_string(msg);
degenerate_backend_set(victim_nodes, number_of_nodes);
- child_exit(1);
+ child_exit(POOL_EXIT_AND_RESTART);
}
else
{
diff --git a/src/utils/error/elog.c b/src/utils/error/elog.c
index cefa93312..9eb0b0fd5 100644
--- a/src/utils/error/elog.c
+++ b/src/utils/error/elog.c
@@ -337,7 +337,7 @@ errstart(int elevel, const char *filename, int lineno,
if(elevel == FATAL && PG_exception_stack == NULL) /* This is startup failure. Take down main process with it */
edata->retcode = POOL_EXIT_FATAL;
else
- edata->retcode = POOL_EXIT_NOFATAL;
+ edata->retcode = POOL_EXIT_AND_RESTART;
if (filename)
{
const char *slash;
diff --git a/src/utils/pool_ssl.c b/src/utils/pool_ssl.c
index 2404e4674..1cd9383c3 100644
--- a/src/utils/pool_ssl.c
+++ b/src/utils/pool_ssl.c
@@ -341,7 +341,7 @@ int pool_ssl_read(POOL_CONNECTION *cp, void *buf, int size) {
ereport(WARNING,
(errmsg("pool_ssl: SSL i/o called but SSL support is not available")));
notice_backend_error(cp->db_node_id);
- child_exit(1);
+ child_exit(POOL_EXIT_AND_RESTART);
return -1; /* never reached */
}
@@ -349,7 +349,7 @@ int pool_ssl_write(POOL_CONNECTION *cp, const void *buf, int size) {
ereport(WARNING,
(errmsg("pool_ssl: SSL i/o called but SSL support is not available")));
notice_backend_error(cp->db_node_id);
- child_exit(1);
+ child_exit(POOL_EXIT_AND_RESTART);
return -1; /* never reached */
}
diff --git a/src/utils/pool_stream.c b/src/utils/pool_stream.c
index ab970bc58..46b2383ac 100644
--- a/src/utils/pool_stream.c
+++ b/src/utils/pool_stream.c
@@ -211,7 +211,7 @@ int pool_read(POOL_CONNECTION *cp, void *buf, int len)
if (pool_config->fail_over_on_backend_error)
{
notice_backend_error(cp->db_node_id);
- child_exit(1);
+ child_exit(POOL_EXIT_AND_RESTART);
/* we are in main process */
ereport(ERROR,
(errmsg("unable to read data from DB node %d",cp->db_node_id),
@@ -248,7 +248,7 @@ int pool_read(POOL_CONNECTION *cp, void *buf, int len)
#ifdef NOT_USED
/* fatal error, notice to parent and exit */
notice_backend_error(IS_MASTER_NODE_ID(cp->db_node_id));
- child_exit(1);
+ child_exit(POOL_EXIT_AND_RESTART);
#endif
}
else
@@ -355,7 +355,7 @@ char *pool_read2(POOL_CONNECTION *cp, int len)
if (pool_config->fail_over_on_backend_error)
{
notice_backend_error(cp->db_node_id);
- child_exit(1);
+ child_exit(POOL_EXIT_AND_RESTART);
/* we are in main process */
ereport(ERROR,
(errmsg("unable to read data from DB node %d",cp->db_node_id),
@@ -387,7 +387,7 @@ char *pool_read2(POOL_CONNECTION *cp, int len)
#ifdef NOT_USED
/* fatal error, notice to parent and exit */
notice_backend_error(IS_MASTER_NODE_ID(cp->db_node_id));
- child_exit(1);
+ child_exit(POOL_EXIT_AND_RESTART);
#endif
}
else
@@ -615,7 +615,7 @@ int pool_flush(POOL_CONNECTION *cp)
(errmsg("unable to flush data to backend"),
errdetail("do not failover because I am the main process")));
- child_exit(1);
+ child_exit(POOL_EXIT_AND_RESTART);
return -1;
}
else
@@ -658,7 +658,7 @@ int pool_flush_noerror(POOL_CONNECTION *cp)
if (pool_config->fail_over_on_backend_error)
{
notice_backend_error(cp->db_node_id);
- child_exit(1);
+ child_exit(POOL_EXIT_AND_RESTART);
ereport(LOG,
(errmsg("unable to flush data to backend"),
errdetail("do not failover because I am the main process")));
@@ -815,7 +815,7 @@ char *pool_read_string(POOL_CONNECTION *cp, int *len, int line)
if (cp->isbackend)
{
notice_backend_error(cp->db_node_id);
- child_exit(1);
+ child_exit(POOL_EXIT_AND_RESTART);
ereport(ERROR,
(errmsg("unable to read data from frontend"),
errdetail("socket read function returned -1")));
diff --git a/src/watchdog/watchdog.c b/src/watchdog/watchdog.c
index 76ab4a2fb..40186d11e 100644
--- a/src/watchdog/watchdog.c
+++ b/src/watchdog/watchdog.c
@@ -2294,7 +2294,7 @@ static void wd_child_signal_handler(void)
if(WEXITSTATUS(status) == POOL_EXIT_FATAL)
ereport(LOG,
(errmsg("watchdog %s process with pid: %d exit with FATAL ERROR.",exiting_process_name, pid)));
- else if(WEXITSTATUS(status) == POOL_EXIT_SUCCESS)
+ else if(WEXITSTATUS(status) == POOL_EXIT_NO_RESTART)
ereport(LOG,
(errmsg("watchdog %s process with pid: %d exit with SUCCESS.",exiting_process_name, pid)));
}
diff --git a/src/watchdog/wd_lifecheck.c b/src/watchdog/wd_lifecheck.c
index b516c3fda..e60431280 100644
--- a/src/watchdog/wd_lifecheck.c
+++ b/src/watchdog/wd_lifecheck.c
@@ -140,7 +140,7 @@ static void reaper(void)
if(WEXITSTATUS(status) == POOL_EXIT_FATAL)
ereport(LOG,
(errmsg("lifecheck child process (%s) with pid: %d exit with FATAL ERROR.",proc_name, pid)));
- else if(WEXITSTATUS(status) == POOL_EXIT_SUCCESS)
+ else if(WEXITSTATUS(status) == POOL_EXIT_NO_RESTART)
{
restart_child = false;
ereport(LOG,
[parent: 67ac541d879b]