pgpool2.git / summary / log / commit / refs
commit 9cc243a625ad17632c496bad8eb8a19e243f7385
Author: Tatsuo Ishii <ishii@postgresql.org>
Date: Thu Apr 21 01:54:55 2016 +0000
Minimize the chance of session disconnection while failover
In addition to pcp_detach_node case, now in admin shutdown and general
error cases are treated same as pcp_detach_node, except the health
check timeout case (retry over case is treated same as normal error
cases). The health check time case is not tested yet though.
For user's convenience, now "show pool_nodes" command shows the
session local load balance node info since this is important for users
in case of failover. If load balance node is not the failed node, the
session will not be affected by failover.
doc/pgpool-en.html | 10 ++--
doc/pgpool-ja.html | 10 ++--
src/include/pcp/libpcp_ext.h | 1 +
src/include/pool.h | 2 +-
src/main/pgpool_main.c | 70 +++++-----------------
src/protocol/pool_connection_pool.c | 2 +-
src/protocol/pool_process_query.c | 29 +++++++--
.../tests/003.failover/create_expected.sql | 11 ++--
src/test/regression/tests/003.failover/test.sh | 2 +-
src/utils/pool_process_reporting.c | 15 ++++-
src/utils/pool_ssl.c | 6 +-
src/utils/pool_stream.c | 14 ++---
12 files changed, 83 insertions(+), 89 deletions(-)
diff --git a/doc/pgpool-en.html b/doc/pgpool-en.html
index 027e23246..06d32a9c6 100644
--- a/doc/pgpool-en.html
+++ b/doc/pgpool-en.html
@@ -3214,16 +3214,16 @@ benchs2=# show pool_status;
<h2 id="pool_nodes">pool_nodes <span class="version">V3.0 -</span></h2>
<p>"SHOW pool_nodes" sends back a list of all configured nodes. It
displays the node id, the hostname, the port, the status, the weight (only meaningful
-if you use the load balancing mode), the role and the SELECT query counts issued to each backend. The possible values in the status column
+if you use the load balancing mode), the role, the SELECT query counts issued to each backend and if the node is for load balancing or not. The possible values in the status column
are explained in the <a href="#pcp_node_info">pcp_node_info reference</a>.
If the hostname is something like "/tmp", that means pgpool-II is connecting to backend by using UNIX domain sockets. The SELECT count does not include internal queries used by pgoool-II. Also the counters are reset to zero upon starting up of pgpool-II.
</p>
<pre>
benchs2=# show pool_nodes;
- node_id | hostname | port | status | lb_weight | role | select_cnt
----------+----------+-------+--------+-----------+---------+------------
- 0 | /tmp | 11002 | 2 | 0.500000 | primary | 9231
- 1 | /tmp | 11003 | 2 | 0.500000 | standby | 9469
+ node_id | hostname | port | status | lb_weight | role | select_cnt | load_balance_node
+---------+----------+-------+--------+-----------+---------+------------+-------------------
+ 0 | /tmp | 11002 | 2 | 0.500000 | primary | 9231 | true
+ 1 | /tmp | 11003 | 2 | 0.500000 | standby | 9469 | false
(2 rows)
</pre>
diff --git a/doc/pgpool-ja.html b/doc/pgpool-ja.html
index 50b2ce36f..0d398b2d8 100644
--- a/doc/pgpool-ja.html
+++ b/doc/pgpool-ja.html
@@ -3365,15 +3365,15 @@ benchs2=# show pool_status;
<h2 id="pool_nodes">pool_nodes <span class="version">V3.0 〜</span></h2>
<p>"SHOW pool_nodes"は、DBノードのリストを表示します。
ホスト名、ポート番号、状態、重み(ロードバランスモードで運用しているときにのみ意味があります)、
-ノードの役割、発行されたSELECTの数が表示されます。
+ノードの役割、発行されたSELECTの数、負荷分散対象のノードかどうかが表示されます。
状態(status)の意味については、<a href="#pcp_node_info">pcp_node_infoリファレンス</a>で説明されています。ホスト名が"/tmp"のように表示される場合、UNIXドメインソケットを使用してpgpool-IIがPosgreSQLに接続していることを意味します。SELECTの数には、pgpool-II内部で発行されるクエリの数は含まれません。また、このカウンタはpgpool-IIがスタートした時に0にリセットされます。
</p>
<pre>
benchs2=# show pool_nodes;
- node_id | hostname | port | status | lb_weight | role | select_cnt
----------+----------+-------+--------+-----------+---------+------------
- 0 | /tmp | 11002 | 2 | 0.500000 | primary | 9231
- 1 | /tmp | 11003 | 2 | 0.500000 | standby | 9469
+ node_id | hostname | port | status | lb_weight | role | select_cnt | load_balance_node
+---------+----------+-------+--------+-----------+---------+------------+-------------------
+ 0 | /tmp | 11002 | 2 | 0.500000 | primary | 9231 | true
+ 1 | /tmp | 11003 | 2 | 0.500000 | standby | 9469 | false
(2 rows)
</pre>
diff --git a/src/include/pcp/libpcp_ext.h b/src/include/pcp/libpcp_ext.h
index 5f4a60465..ed86e7bee 100644
--- a/src/include/pcp/libpcp_ext.h
+++ b/src/include/pcp/libpcp_ext.h
@@ -144,6 +144,7 @@ typedef struct {
char lb_weight[POOLCONFIG_MAXWEIGHTLEN+1];
char role[POOLCONFIG_MAXWEIGHTLEN+1];
char select[POOLCONFIG_MAXWEIGHTLEN+1];
+ char load_balance_node[POOLCONFIG_MAXWEIGHTLEN+1];
} POOL_REPORT_NODES;
/* processes report struct */
diff --git a/src/include/pool.h b/src/include/pool.h
index 864b5250a..30759e0f4 100644
--- a/src/include/pool.h
+++ b/src/include/pool.h
@@ -536,7 +536,7 @@ extern POOL_STATUS ErrorResponse(POOL_CONNECTION *frontend,
extern void NoticeResponse(POOL_CONNECTION *frontend,
POOL_CONNECTION_POOL *backend);
-extern void notice_backend_error(int node_id);
+extern void notice_backend_error(int node_id, bool switch_over);
extern void degenerate_backend_set(int *node_id_set, int count, bool switch_over);
extern bool degenerate_backend_set_ex(int *node_id_set, int count, bool error, bool test_only, bool switch_over);
extern void promote_backend(int node_id);
diff --git a/src/main/pgpool_main.c b/src/main/pgpool_main.c
index c320993ad..30c951cac 100644
--- a/src/main/pgpool_main.c
+++ b/src/main/pgpool_main.c
@@ -539,10 +539,17 @@ process_backend_health_check_failure(int health_check_node_id, int retrycnt)
}
else
{
+ /*
+ * If health check timer expired, then it is possible that there
+ * was a communitication path problem. In that case we need to do
+ * full restarting of child process.
+ */
+ bool partial_failover = health_check_timer_expired? false:true;
+
ereport(LOG,
(errmsg("setting backend node %d status to NODE DOWN", health_check_node_id)));
health_check_timer_expired = 0;
- degenerate_backend_set(&health_check_node_id, 1, false);
+ degenerate_backend_set(&health_check_node_id, 1, partial_failover);
return 2;
/* need to distribute this info to children ??*/
}
@@ -1043,7 +1050,11 @@ static void terminate_all_childrens()
POOL_SETMASK(&UnBlockSig);
}
-void notice_backend_error(int node_id)
+/*
+ * Reuest failover. If "switch_over" is false, request all existing sessions
+ * restarting.
+ */
+void notice_backend_error(int node_id, bool switch_over)
{
int n = node_id;
@@ -1054,7 +1065,7 @@ void notice_backend_error(int node_id)
}
else
{
- degenerate_backend_set(&n, 1, false);
+ degenerate_backend_set(&n, 1, switch_over);
}
}
@@ -1649,57 +1660,6 @@ static void failover(void)
(errmsg("failover: no valid backends node found")));
}
- /*
- * Before we tried to minimize restarting pgpool to protect existing
- * connections from clients to pgpool children. What we did here was,
- * if children other than master went down, we did not fail over.
- * This is wrong. Think about following scenario. If someone
- * accidentally plugs out the network cable, the TCP/IP stack keeps
- * retrying for long time (typically 2 hours). The only way to stop
- * the retry is restarting the process. Bottom line is, we need to
- * restart all children in any case. See pgpool-general list posting
- * "TCP connections are *not* closed when a backend timeout" on Jul 13
- * 2008 for more details.
- */
-#ifdef NOT_USED
- else
- {
- if (Req_info->master_node_id == new_master && *InRecovery == RECOVERY_INIT)
- {
- ereport(LOG,
- (errmsg("failover_handler: do not restart pgpool. same master node %d was selected", new_master)));
- if (reqkind == NODE_UP_REQUEST)
- {
- ereport(LOG,
- (errmsg("failback done. reconnect host %s(%d)",
- BACKEND_INFO(node_id).backend_hostname,
- BACKEND_INFO(node_id).backend_port)));
- }
- else
- {
- ereport(LOG,
- (errmsg("failover done. shutdown host %s(%d)",
- BACKEND_INFO(node_id).backend_hostname,
- BACKEND_INFO(node_id).backend_port)));
- }
-
- /* exec failover_command */
- for (i = 0; i < pool_config->backend_desc->num_backends; i++)
- {
- if (nodes[i])
- trigger_failover_command(i, pool_config->failover_command);
- }
-
- switching = 0;
- Req_info->switching = false;
- kill(pcp_pid, SIGUSR2);
- switching = 0;
- Req_info->switching = false;
- continue;
- }
- }
-#endif
-
ereport(DEBUG1, (errmsg("failover/failback request details: STREAM: %d reqkind: %d detail: %x node_id: %d",
STREAM, reqkind, request_details & REQ_DETAIL_SWITCHOVER,
node_id)));
@@ -1979,6 +1939,8 @@ static void failover(void)
process_info[i].start_time = time(NULL);
}
}
+ else
+ process_info[i].need_to_restart = 1;
}
}
diff --git a/src/protocol/pool_connection_pool.c b/src/protocol/pool_connection_pool.c
index a7de1b16a..b7cc946e4 100644
--- a/src/protocol/pool_connection_pool.c
+++ b/src/protocol/pool_connection_pool.c
@@ -847,7 +847,7 @@ static POOL_CONNECTION_POOL *new_connection(POOL_CONNECTION_POOL *p)
*/
if (pool_config->fail_over_on_backend_error)
{
- notice_backend_error(i);
+ notice_backend_error(i, true);
ereport(FATAL,
(errmsg("failed to create a backend connection"),
errdetail("executing failover on backend")));
diff --git a/src/protocol/pool_process_query.c b/src/protocol/pool_process_query.c
index 19f6b8258..bfee3c24f 100644
--- a/src/protocol/pool_process_query.c
+++ b/src/protocol/pool_process_query.c
@@ -4584,13 +4584,30 @@ SELECT_RETRY:
ereport(LOG,
(errmsg("reading and processing packets"),
errdetail("postmaster on DB node %d was shutdown by administrative command", i)));
- /* detach backend node. */
- was_error = 1;
- if (!VALID_BACKEND(i))
+
+ /*
+ * If shutdown node is not primary nor load balance node,
+ * we do not need to trigger failover.
+ */
+ if (STREAM &&
+ (i == PRIMARY_NODE_ID || i == backend->info->load_balancing_node))
+ {
+ /* detach backend node. */
+ was_error = 1;
+ if (!VALID_BACKEND(i))
+ break;
+ notice_backend_error(i, true);
+ sleep(5);
break;
- notice_backend_error(i);
- sleep(5);
- break;
+ }
+
+ /*
+ * Just set local status to down.
+ */
+ else
+ {
+ *(my_backend_status[i]) = CON_DOWN;
+ }
}
else if (r < 0)
{
diff --git a/src/test/regression/tests/003.failover/create_expected.sql b/src/test/regression/tests/003.failover/create_expected.sql
index 35887adba..342b06a3a 100644
--- a/src/test/regression/tests/003.failover/create_expected.sql
+++ b/src/test/regression/tests/003.failover/create_expected.sql
@@ -6,14 +6,15 @@ CREATE TEMP TABLE tmp (
lb_weight text,
role text,
select_cnt text,
+ load_balance_node text,
mode text);
INSERT INTO tmp VALUES
-('0',:dir,'11002','2','0.500000','primary','0','s'),
-('1',:dir,'11003','3','0.500000','standby','0','s'),
-('0',:dir,'11002','2','0.500000','master','0','r'),
-('1',:dir,'11003','3','0.500000','slave','0','r');
+('0',:dir,'11002','2','0.500000','primary','0','false','s'),
+('1',:dir,'11003','3','0.500000','standby','0','false','s'),
+('0',:dir,'11002','2','0.500000','master','0','false','r'),
+('1',:dir,'11003','3','0.500000','slave','0','false','r');
-SELECT node_id,hostname,port,status,lb_weight,role,select_cnt
+SELECT node_id,hostname,port,status,lb_weight,role,select_cnt,load_balance_node
FROM tmp
WHERE mode = :mode
diff --git a/src/test/regression/tests/003.failover/test.sh b/src/test/regression/tests/003.failover/test.sh
index 85ba3fc35..5ecd5998c 100755
--- a/src/test/regression/tests/003.failover/test.sh
+++ b/src/test/regression/tests/003.failover/test.sh
@@ -27,7 +27,7 @@ do
# trrigger failover
$PG_CTL -D data1 -m f stop
wait_for_pgpool_startup
- $PSQL -c "show pool_nodes" test > result
+ $PSQL -c "show pool_nodes" test |sed -e 's/true/false/'> result
# check the output of "show pool_nodes".
LANG=C $PSQL -f ../create_expected.sql -v mode="'$mode'" -v dir="'$PGSOCKET_DIR'" test | tail -n 6 > expected
diff --git a/src/utils/pool_process_reporting.c b/src/utils/pool_process_reporting.c
index e65648b75..57735f855 100644
--- a/src/utils/pool_process_reporting.c
+++ b/src/utils/pool_process_reporting.c
@@ -1032,6 +1032,7 @@ POOL_REPORT_NODES* get_nodes(int *nrows)
int i;
POOL_REPORT_NODES* nodes = palloc(NUM_BACKENDS * sizeof(POOL_REPORT_NODES));
BackendInfo *bi = NULL;
+ POOL_SESSION_CONTEXT *session_context = pool_get_session_context(false);
for (i = 0; i < NUM_BACKENDS; i++)
{
@@ -1043,6 +1044,8 @@ POOL_REPORT_NODES* get_nodes(int *nrows)
snprintf(nodes[i].status, POOLCONFIG_MAXSTATLEN, "%d", bi->backend_status);
snprintf(nodes[i].lb_weight, POOLCONFIG_MAXWEIGHTLEN, "%f", bi->backend_weight/RAND_MAX);
snprintf(nodes[i].select, POOLCONFIG_MAXWEIGHTLEN, "%lld", stat_get_select_count(i));
+ snprintf(nodes[i].load_balance_node, POOLCONFIG_MAXWEIGHTLEN, "%s",
+ (session_context->load_balance_node_id == i)? "true":"false");
if (MASTER_SLAVE && !strcmp(pool_config->master_slave_sub_mode, MODE_STREAMREP))
if (i == REAL_PRIMARY_NODE_ID)
@@ -1065,7 +1068,7 @@ POOL_REPORT_NODES* get_nodes(int *nrows)
void nodes_reporting(POOL_CONNECTION *frontend, POOL_CONNECTION_POOL *backend)
{
- static char *field_names[] = {"node_id","hostname", "port", "status", "lb_weight", "role", "select_cnt"};
+ static char *field_names[] = {"node_id","hostname", "port", "status", "lb_weight", "role", "select_cnt", "load_balance_node"};
short num_fields = sizeof(field_names)/sizeof(char *);
int i;
short s;
@@ -1122,6 +1125,11 @@ void nodes_reporting(POOL_CONNECTION *frontend, POOL_CONNECTION_POOL *backend)
hsize = htonl(size+4);
pool_write(frontend, &hsize, sizeof(hsize));
pool_write(frontend, nodes[i].select, size);
+
+ size = strlen(nodes[i].load_balance_node);
+ hsize = htonl(size+4);
+ pool_write(frontend, &hsize, sizeof(hsize));
+ pool_write(frontend, nodes[i].load_balance_node, size);
}
}
else
@@ -1138,6 +1146,7 @@ void nodes_reporting(POOL_CONNECTION *frontend, POOL_CONNECTION_POOL *backend)
len += 4 + strlen(nodes[i].lb_weight); /* int32 + data; */
len += 4 + strlen(nodes[i].role); /* int32 + data; */
len += 4 + strlen(nodes[i].select); /* int32 + data; */
+ len += 4 + strlen(nodes[i].load_balance_node); /* int32 + data; */
len = htonl(len);
pool_write(frontend, &len, sizeof(len));
s = htons(num_fields);
@@ -1170,6 +1179,10 @@ void nodes_reporting(POOL_CONNECTION *frontend, POOL_CONNECTION_POOL *backend)
len = htonl(strlen(nodes[i].select));
pool_write(frontend, &len, sizeof(len));
pool_write(frontend, nodes[i].select, strlen(nodes[i].select));
+
+ len = htonl(strlen(nodes[i].load_balance_node));
+ pool_write(frontend, &len, sizeof(len));
+ pool_write(frontend, nodes[i].load_balance_node, strlen(nodes[i].load_balance_node));
}
}
diff --git a/src/utils/pool_ssl.c b/src/utils/pool_ssl.c
index 1cd9383c3..82ba82e55 100644
--- a/src/utils/pool_ssl.c
+++ b/src/utils/pool_ssl.c
@@ -5,7 +5,7 @@
* pgpool: a language independent connection pool server for PostgreSQL
* written by Tatsuo Ishii
*
- * Copyright (c) 2003-2013 PgPool Global Development Group
+ * Copyright (c) 2003-2016 PgPool Global Development Group
*
* Permission to use, copy, modify, and distribute this software and
* its documentation for any purpose and without fee is hereby
@@ -340,7 +340,7 @@ void pool_ssl_close(POOL_CONNECTION *cp) { return; }
int pool_ssl_read(POOL_CONNECTION *cp, void *buf, int size) {
ereport(WARNING,
(errmsg("pool_ssl: SSL i/o called but SSL support is not available")));
- notice_backend_error(cp->db_node_id);
+ notice_backend_error(cp->db_node_id, true);
child_exit(POOL_EXIT_AND_RESTART);
return -1; /* never reached */
}
@@ -348,7 +348,7 @@ int pool_ssl_read(POOL_CONNECTION *cp, void *buf, int size) {
int pool_ssl_write(POOL_CONNECTION *cp, const void *buf, int size) {
ereport(WARNING,
(errmsg("pool_ssl: SSL i/o called but SSL support is not available")));
- notice_backend_error(cp->db_node_id);
+ notice_backend_error(cp->db_node_id, true);
child_exit(POOL_EXIT_AND_RESTART);
return -1; /* never reached */
}
diff --git a/src/utils/pool_stream.c b/src/utils/pool_stream.c
index 46b2383ac..72690541f 100644
--- a/src/utils/pool_stream.c
+++ b/src/utils/pool_stream.c
@@ -210,7 +210,7 @@ int pool_read(POOL_CONNECTION *cp, void *buf, int len)
/* if fail_over_on_backend_error is true, then trigger failover */
if (pool_config->fail_over_on_backend_error)
{
- notice_backend_error(cp->db_node_id);
+ notice_backend_error(cp->db_node_id, true);
child_exit(POOL_EXIT_AND_RESTART);
/* we are in main process */
ereport(ERROR,
@@ -247,7 +247,7 @@ int pool_read(POOL_CONNECTION *cp, void *buf, int len)
#ifdef NOT_USED
/* fatal error, notice to parent and exit */
- notice_backend_error(IS_MASTER_NODE_ID(cp->db_node_id));
+ notice_backend_error(IS_MASTER_NODE_ID(cp->db_node_id, true));
child_exit(POOL_EXIT_AND_RESTART);
#endif
}
@@ -354,7 +354,7 @@ char *pool_read2(POOL_CONNECTION *cp, int len)
/* if fail_over_on_backend_error is true, then trigger failover */
if (pool_config->fail_over_on_backend_error)
{
- notice_backend_error(cp->db_node_id);
+ notice_backend_error(cp->db_node_id, true);
child_exit(POOL_EXIT_AND_RESTART);
/* we are in main process */
ereport(ERROR,
@@ -386,7 +386,7 @@ char *pool_read2(POOL_CONNECTION *cp, int len)
#ifdef NOT_USED
/* fatal error, notice to parent and exit */
- notice_backend_error(IS_MASTER_NODE_ID(cp->db_node_id));
+ notice_backend_error(IS_MASTER_NODE_ID(cp->db_node_id, true));
child_exit(POOL_EXIT_AND_RESTART);
#endif
}
@@ -610,7 +610,7 @@ int pool_flush(POOL_CONNECTION *cp)
/* if fail_over_on_backend_error is true, then trigger failover */
if (pool_config->fail_over_on_backend_error)
{
- notice_backend_error(cp->db_node_id);
+ notice_backend_error(cp->db_node_id, true);
ereport(LOG,
(errmsg("unable to flush data to backend"),
errdetail("do not failover because I am the main process")));
@@ -657,7 +657,7 @@ int pool_flush_noerror(POOL_CONNECTION *cp)
/* if fail_over_on_backend_erro is true, then trigger failover */
if (pool_config->fail_over_on_backend_error)
{
- notice_backend_error(cp->db_node_id);
+ notice_backend_error(cp->db_node_id, true);
child_exit(POOL_EXIT_AND_RESTART);
ereport(LOG,
(errmsg("unable to flush data to backend"),
@@ -814,7 +814,7 @@ char *pool_read_string(POOL_CONNECTION *cp, int *len, int line)
cp->socket_state = POOL_SOCKET_ERROR;
if (cp->isbackend)
{
- notice_backend_error(cp->db_node_id);
+ notice_backend_error(cp->db_node_id, true);
child_exit(POOL_EXIT_AND_RESTART);
ereport(ERROR,
(errmsg("unable to read data from frontend"),
[parent: 883d761b406e]