pgpool2.git / summary / log / commit / refs
commit 9f12b15f322e9094873ff6039b7d4de7cd019baa
Author: Muhammad Usama <m.usama@gmail.com>
Date: Thu Nov 09 13:46:52 2017 +0000
Add some watchdog test cases
src/include/pool_config.h | 2 +-
.../master.conf | 24 +++
.../standby.conf | 26 +++
.../standby2.conf | 26 +++
.../test.sh | 181 ++++++++++++++++++
.../master.conf | 24 +++
.../standby.conf | 26 +++
.../standby2.conf | 26 +++
.../test.sh | 202 +++++++++++++++++++++
.../014.watchdoc_test_quorum_bypass/master.conf | 24 +++
.../tests/014.watchdoc_test_quorum_bypass/test.sh | 81 +++++++++
.../master.conf | 24 +++
.../standby.conf | 26 +++
.../standby2.conf | 26 +++
.../test.sh | 186 +++++++++++++++++++
src/watchdog/watchdog.c | 4 +-
16 files changed, 905 insertions(+), 3 deletions(-)
diff --git a/src/include/pool_config.h b/src/include/pool_config.h
index 22ea360de..3366e6f70 100644
--- a/src/include/pool_config.h
+++ b/src/include/pool_config.h
@@ -300,7 +300,7 @@ typedef struct {
* add for watchdog
*/
bool use_watchdog; /* Enables watchdog */
- bool failover_when_quorum_exists; /* Do failover only when wd cluster holds he quorum */
+ bool failover_when_quorum_exists; /* Do failover only when wd cluster holds the quorum */
bool failover_require_consensus; /* Only do failover when majority aggrees */
bool allow_multiple_failover_requests_from_node; /* One Pgpool-II node can send multiple
* failover requests to build consensus
diff --git a/src/test/regression/tests/012.watchdoc_test_failover_when_quorum_exists/master.conf b/src/test/regression/tests/012.watchdoc_test_failover_when_quorum_exists/master.conf
new file mode 100644
index 000000000..0b83c7192
--- /dev/null
+++ b/src/test/regression/tests/012.watchdoc_test_failover_when_quorum_exists/master.conf
@@ -0,0 +1,24 @@
+# master watchdog
+num_init_children = 4
+use_watchdog = on
+failover_when_quorum_exists = true
+failover_require_consensus = false
+allow_multiple_failover_requests_from_node = false
+wd_interval = 1
+wd_priority = 5
+wd_hostname = 'localhost'
+wd_port = 21004
+wd_heartbeat_port = 21005
+
+heartbeat_destination0 = 'localhost'
+heartbeat_destination_port0 = 21105
+heartbeat_destination1 = 'localhost'
+heartbeat_destination_port1 = 21205
+
+other_pgpool_hostname0 = 'localhost'
+other_pgpool_port0 = 11100
+other_wd_port0 = 21104
+
+other_pgpool_hostname1 = 'localhost'
+other_pgpool_port1 = 11200
+other_wd_port1 = 21204
diff --git a/src/test/regression/tests/012.watchdoc_test_failover_when_quorum_exists/standby.conf b/src/test/regression/tests/012.watchdoc_test_failover_when_quorum_exists/standby.conf
new file mode 100644
index 000000000..e03412d85
--- /dev/null
+++ b/src/test/regression/tests/012.watchdoc_test_failover_when_quorum_exists/standby.conf
@@ -0,0 +1,26 @@
+# standby watchdog
+num_init_children = 4
+port = 11100
+pcp_port = 11105
+use_watchdog = on
+failover_when_quorum_exists = true
+failover_require_consensus = false
+allow_multiple_failover_requests_from_node = false
+wd_interval = 1
+wd_priority = 3
+wd_hostname = 'localhost'
+wd_port = 21104
+wd_heartbeat_port = 21105
+heartbeat_destination0 = 'localhost'
+heartbeat_destination_port0 = 21005
+heartbeat_destination1 = 'localhost'
+heartbeat_destination_port1 = 21205
+
+other_pgpool_hostname0 = 'localhost'
+other_pgpool_port0 = 11000
+other_wd_port0 = 21004
+
+other_pgpool_hostname1 = 'localhost'
+other_pgpool_port1 = 11200
+other_wd_port1 = 21204
+
diff --git a/src/test/regression/tests/012.watchdoc_test_failover_when_quorum_exists/standby2.conf b/src/test/regression/tests/012.watchdoc_test_failover_when_quorum_exists/standby2.conf
new file mode 100644
index 000000000..f28bd9da3
--- /dev/null
+++ b/src/test/regression/tests/012.watchdoc_test_failover_when_quorum_exists/standby2.conf
@@ -0,0 +1,26 @@
+# standby2 watchdog
+num_init_children = 4
+port = 11200
+pcp_port = 11205
+use_watchdog = on
+failover_when_quorum_exists = true
+failover_require_consensus = false
+allow_multiple_failover_requests_from_node = false
+wd_interval = 1
+wd_priority = 1
+wd_hostname = 'localhost'
+wd_port = 21204
+wd_heartbeat_port = 21205
+heartbeat_destination0 = 'localhost'
+heartbeat_destination_port0 = 21005
+heartbeat_destination1 = 'localhost'
+heartbeat_destination_port1 = 21105
+
+other_pgpool_hostname0 = 'localhost'
+other_pgpool_port0 = 11000
+other_wd_port0 = 21004
+
+other_pgpool_hostname1 = 'localhost'
+other_pgpool_port1 = 11100
+other_wd_port1 = 21104
+
diff --git a/src/test/regression/tests/012.watchdoc_test_failover_when_quorum_exists/test.sh b/src/test/regression/tests/012.watchdoc_test_failover_when_quorum_exists/test.sh
new file mode 100755
index 000000000..6fa062093
--- /dev/null
+++ b/src/test/regression/tests/012.watchdoc_test_failover_when_quorum_exists/test.sh
@@ -0,0 +1,181 @@
+#!/usr/bin/env bash
+#-------------------------------------------------------------------
+# test script for watchdog
+#
+# Please note that to successfully run the test, "HEALTHCHECK_DEBUG"
+# must be defined before compiling main/health_check.c.
+#
+# test failover_when_quorum_exists
+#
+source $TESTLIBS
+MASTER_DIR=master
+STANDBY_DIR=standby
+STANDBY2_DIR=standby2
+num_tests=5
+success_count=0
+PSQL=$PGBIN/psql
+PG_CTL=$PGBIN/pg_ctl
+
+rm -fr $MASTER_DIR
+rm -fr $STANDBY_DIR
+rm -fr $STANDBY2_DIR
+
+mkdir $MASTER_DIR
+mkdir $STANDBY_DIR
+mkdir $STANDBY2_DIR
+
+
+# dir in master directory
+cd $MASTER_DIR
+
+# create master environment
+echo -n "creating master pgpool and PostgreSQL clusters..."
+$PGPOOL_SETUP -m s -n 2 -p 11000|| exit 1
+echo "master setup done."
+
+
+# copy the configurations from master to standby
+cp -r etc ../$STANDBY_DIR/
+
+# copy the configurations from master to standby2
+cp -r etc ../$STANDBY2_DIR/
+
+source ./bashrc.ports
+cat ../master.conf >> etc/pgpool.conf
+
+./startall
+wait_for_pgpool_startup
+
+
+# back to test root dir
+cd ..
+
+
+# create standby environment but do not start pgpool
+
+mkdir $STANDBY_DIR/log
+echo -n "creating standby pgpool..."
+cat standby.conf >> $STANDBY_DIR/etc/pgpool.conf
+# since we are using the same pgpool-II conf as of master. so change the pid file path in standby pgpool conf
+echo "pid_file_name = '$PWD/pgpool2.pid'" >> $STANDBY_DIR/etc/pgpool.conf
+echo "logdir = $STANDBY_DIR/log" >> $STANDBY_DIR/etc/pgpool.conf
+# start the stnadby pgpool-II by hand
+#$PGPOOL_INSTALL_DIR/bin/pgpool -D -n -f $STANDBY_DIR/etc/pgpool.conf -F $STANDBY_DIR/etc/pcp.conf -a $STANDBY_DIR/etc/pool_hba.conf > $STANDBY_DIR/log/pgpool.log 2>&1 &
+
+# create standby2 environment but do not start pgpool
+
+mkdir $STANDBY2_DIR/log
+echo -n "creating standby2 pgpool..."
+cat standby2.conf >> $STANDBY2_DIR/etc/pgpool.conf
+# since we are using the same pgpool-II conf as of master. so change the pid file path in standby pgpool conf
+echo "pid_file_name = '$PWD/pgpool3.pid'" >> $STANDBY2_DIR/etc/pgpool.conf
+echo "logdir = $STANDBY2_DIR/log" >> $STANDBY2_DIR/etc/pgpool.conf
+# start the stnadby pgpool-II by hand
+#$PGPOOL_INSTALL_DIR/bin/pgpool -D -n -f $STANDBY2_DIR/etc/pgpool.conf -F $STANDBY2_DIR/etc/pcp.conf -a $STANDBY2_DIR/etc/pool_hba.conf > $STANDBY2_DIR/log/pgpool.log 2>&1 &
+
+# First test check if both pgpool-II have found their correct place in watchdog cluster.
+echo "Waiting for the pgpool master..."
+for i in 1 2 3 4 5 6 7 8 9 10
+do
+ grep "I am the cluster leader node" $MASTER_DIR/log/pgpool.log > /dev/null 2>&1
+ if [ $? = 0 ];then
+ success_count=$(( success_count + 1 ))
+ echo "Master brought up successfully."
+ break;
+ fi
+ echo "[check] $i times"
+ sleep 2
+done
+
+
+# raise an artificial communication error on master for DB node 1
+echo "1 down" > $MASTER_DIR/log/backend_down_request
+echo "Checking if the Master rejects the failover because quorum is not present..."
+for i in 1 2 3 4 5 6 7 8 9 10
+do
+ grep -i "Rejecting the failover request" $MASTER_DIR/log/pgpool.log
+ if [ $? = 0 ];then
+ success_count=$(( success_count + 1 ))
+ echo "Fake DB error detected. and Failover rejected because of absence of quorum"
+ break;
+ fi
+ echo "[check] $i times"
+ sleep 2
+done
+
+# now start other Pgpool-II nodes to complete the quorum
+
+# start the stnadby pgpool-II by hand
+$PGPOOL_INSTALL_DIR/bin/pgpool -D -n -f $STANDBY_DIR/etc/pgpool.conf -F $STANDBY_DIR/etc/pcp.conf -a $STANDBY_DIR/etc/pool_hba.conf > $STANDBY_DIR/log/pgpool.log 2>&1 &
+# start the second stnadby pgpool-II by hand
+$PGPOOL_INSTALL_DIR/bin/pgpool -D -n -f $STANDBY2_DIR/etc/pgpool.conf -F $STANDBY2_DIR/etc/pcp.conf -a $STANDBY2_DIR/etc/pool_hba.conf > $STANDBY2_DIR/log/pgpool.log 2>&1 &
+
+# now check if standby1 has successfully joined connected to the master.
+echo "Waiting for the standby1 to join cluster..."
+for i in 1 2 3 4 5 6 7 8 9 10
+do
+ grep "successfully joined the watchdog cluster as standby node" $STANDBY_DIR/log/pgpool.log > /dev/null 2>&1
+ if [ $? = 0 ];then
+ success_count=$(( success_count + 1 ))
+ echo "Standby successfully connected."
+ break;
+ fi
+ echo "[check] $i times"
+ sleep 2
+done
+
+# now check if standby2 has successfully joined connected to the master.
+echo "Waiting for the standby2 to join cluster..."
+for i in 1 2 3 4 5 6 7 8 9 10
+do
+ grep "successfully joined the watchdog cluster as standby node" $STANDBY2_DIR/log/pgpool.log > /dev/null 2>&1
+ if [ $? = 0 ];then
+ success_count=$(( success_count + 1 ))
+ echo "Standby2 successfully connected."
+ break;
+ fi
+ echo "[check] $i times"
+ sleep 2
+done
+
+# raise an artificial communication again to check if failover is executed this time
+echo "1 down" > $MASTER_DIR/log/backend_down_request
+#give some time to pgpool-II to execute failover
+sleep 5
+# check to see if all Pgpool-II agrees that the failover request is
+# executed
+echo "Checking if all Pgpool-II agrees that the failover request is executed"
+for i in 1 2 3 4 5 6 7 8 9 10
+do
+ n=0
+ for p in 11000 11100 11200
+ do
+ $PSQL -p $p -c "show pool_nodes" test|grep standby|grep down >/dev/null 2>&1
+ if [ $? = 0 ];then
+ n=$(( n + 1))
+ fi
+ done
+ if [ $n -eq 3 ];then
+ success_count=$(( success_count + 1 ))
+ echo "All Pgpool-II agrees that the failover request is executed"
+ break;
+ fi
+ echo "[check] $i times"
+ sleep 2
+done
+
+
+# we are done. Just stop the standby pgpool-II
+$PGPOOL_INSTALL_DIR/bin/pgpool -f $STANDBY_DIR/etc/pgpool.conf -m f stop
+$PGPOOL_INSTALL_DIR/bin/pgpool -f $STANDBY2_DIR/etc/pgpool.conf -m f stop
+cd master
+./shutdownall
+
+echo "$success_count out of $num_tests successfull";
+
+if test $success_count -eq $num_tests
+then
+ exit 0
+fi
+
+exit 1
diff --git a/src/test/regression/tests/013.watchdoc_test_failover_require_consensus/master.conf b/src/test/regression/tests/013.watchdoc_test_failover_require_consensus/master.conf
new file mode 100644
index 000000000..58e6f4bad
--- /dev/null
+++ b/src/test/regression/tests/013.watchdoc_test_failover_require_consensus/master.conf
@@ -0,0 +1,24 @@
+# master watchdog
+num_init_children = 4
+use_watchdog = on
+failover_when_quorum_exists = true
+failover_require_consensus = true
+allow_multiple_failover_requests_from_node = false
+wd_interval = 1
+wd_priority = 5
+wd_hostname = 'localhost'
+wd_port = 21004
+wd_heartbeat_port = 21005
+
+heartbeat_destination0 = 'localhost'
+heartbeat_destination_port0 = 21105
+heartbeat_destination1 = 'localhost'
+heartbeat_destination_port1 = 21205
+
+other_pgpool_hostname0 = 'localhost'
+other_pgpool_port0 = 11100
+other_wd_port0 = 21104
+
+other_pgpool_hostname1 = 'localhost'
+other_pgpool_port1 = 11200
+other_wd_port1 = 21204
diff --git a/src/test/regression/tests/013.watchdoc_test_failover_require_consensus/standby.conf b/src/test/regression/tests/013.watchdoc_test_failover_require_consensus/standby.conf
new file mode 100644
index 000000000..710cbb7ae
--- /dev/null
+++ b/src/test/regression/tests/013.watchdoc_test_failover_require_consensus/standby.conf
@@ -0,0 +1,26 @@
+# standby watchdog
+num_init_children = 4
+port = 11100
+pcp_port = 11105
+use_watchdog = on
+failover_when_quorum_exists = true
+failover_require_consensus = true
+allow_multiple_failover_requests_from_node = false
+wd_interval = 1
+wd_priority = 3
+wd_hostname = 'localhost'
+wd_port = 21104
+wd_heartbeat_port = 21105
+heartbeat_destination0 = 'localhost'
+heartbeat_destination_port0 = 21005
+heartbeat_destination1 = 'localhost'
+heartbeat_destination_port1 = 21205
+
+other_pgpool_hostname0 = 'localhost'
+other_pgpool_port0 = 11000
+other_wd_port0 = 21004
+
+other_pgpool_hostname1 = 'localhost'
+other_pgpool_port1 = 11200
+other_wd_port1 = 21204
+
diff --git a/src/test/regression/tests/013.watchdoc_test_failover_require_consensus/standby2.conf b/src/test/regression/tests/013.watchdoc_test_failover_require_consensus/standby2.conf
new file mode 100644
index 000000000..f44714133
--- /dev/null
+++ b/src/test/regression/tests/013.watchdoc_test_failover_require_consensus/standby2.conf
@@ -0,0 +1,26 @@
+# standby2 watchdog
+num_init_children = 4
+port = 11200
+pcp_port = 11205
+use_watchdog = on
+failover_when_quorum_exists = true
+failover_require_consensus = true
+allow_multiple_failover_requests_from_node = false
+wd_interval = 1
+wd_priority = 1
+wd_hostname = 'localhost'
+wd_port = 21204
+wd_heartbeat_port = 21205
+heartbeat_destination0 = 'localhost'
+heartbeat_destination_port0 = 21005
+heartbeat_destination1 = 'localhost'
+heartbeat_destination_port1 = 21105
+
+other_pgpool_hostname0 = 'localhost'
+other_pgpool_port0 = 11000
+other_wd_port0 = 21004
+
+other_pgpool_hostname1 = 'localhost'
+other_pgpool_port1 = 11100
+other_wd_port1 = 21104
+
diff --git a/src/test/regression/tests/013.watchdoc_test_failover_require_consensus/test.sh b/src/test/regression/tests/013.watchdoc_test_failover_require_consensus/test.sh
new file mode 100755
index 000000000..f4249839f
--- /dev/null
+++ b/src/test/regression/tests/013.watchdoc_test_failover_require_consensus/test.sh
@@ -0,0 +1,202 @@
+#!/usr/bin/env bash
+#-------------------------------------------------------------------
+# test script for watchdog
+#
+# Please note that to successfully run the test, "HEALTHCHECK_DEBUG"
+# must be defined before compiling main/health_check.c.
+#
+# test failover_require_consensus
+#
+source $TESTLIBS
+MASTER_DIR=master
+STANDBY_DIR=standby
+STANDBY2_DIR=standby2
+num_tests=7
+success_count=0
+PSQL=$PGBIN/psql
+PG_CTL=$PGBIN/pg_ctl
+
+rm -fr $MASTER_DIR
+rm -fr $STANDBY_DIR
+rm -fr $STANDBY2_DIR
+
+mkdir $MASTER_DIR
+mkdir $STANDBY_DIR
+mkdir $STANDBY2_DIR
+
+
+# dir in master directory
+cd $MASTER_DIR
+
+# create master environment
+echo -n "creating master pgpool and PostgreSQL clusters..."
+$PGPOOL_SETUP -m s -n 2 -p 11000|| exit 1
+echo "master setup done."
+
+
+# copy the configurations from master to standby
+cp -r etc ../$STANDBY_DIR/
+
+# copy the configurations from master to standby2
+cp -r etc ../$STANDBY2_DIR/
+
+source ./bashrc.ports
+cat ../master.conf >> etc/pgpool.conf
+
+./startall
+wait_for_pgpool_startup
+
+
+# back to test root dir
+cd ..
+
+
+# create standby environment and start pgpool
+
+mkdir $STANDBY_DIR/log
+echo -n "creating standby pgpool..."
+cat standby.conf >> $STANDBY_DIR/etc/pgpool.conf
+# since we are using the same pgpool-II conf as of master. so change the pid file path in standby pgpool conf
+echo "pid_file_name = '$PWD/pgpool2.pid'" >> $STANDBY_DIR/etc/pgpool.conf
+echo "logdir = $STANDBY_DIR/log" >> $STANDBY_DIR/etc/pgpool.conf
+# start the stnadby pgpool-II by hand
+$PGPOOL_INSTALL_DIR/bin/pgpool -D -n -f $STANDBY_DIR/etc/pgpool.conf -F $STANDBY_DIR/etc/pcp.conf -a $STANDBY_DIR/etc/pool_hba.conf > $STANDBY_DIR/log/pgpool.log 2>&1 &
+
+# create standby2 environment but do not start pgpool
+mkdir $STANDBY2_DIR/log
+echo -n "creating standby2 pgpool..."
+cat standby2.conf >> $STANDBY2_DIR/etc/pgpool.conf
+# since we are using the same pgpool-II conf as of master. so change the pid file path in standby pgpool conf
+echo "pid_file_name = '$PWD/pgpool3.pid'" >> $STANDBY2_DIR/etc/pgpool.conf
+echo "logdir = $STANDBY2_DIR/log" >> $STANDBY2_DIR/etc/pgpool.conf
+# start the stnadby pgpool-II by hand
+$PGPOOL_INSTALL_DIR/bin/pgpool -D -n -f $STANDBY2_DIR/etc/pgpool.conf -F $STANDBY2_DIR/etc/pcp.conf -a $STANDBY2_DIR/etc/pool_hba.conf > $STANDBY2_DIR/log/pgpool.log 2>&1 &
+
+# First test check if both pgpool-II have found their correct place in watchdog cluster.
+echo "Waiting for the pgpool master..."
+for i in 1 2 3 4 5 6 7 8 9 10
+do
+ grep "I am the cluster leader node" $MASTER_DIR/log/pgpool.log > /dev/null 2>&1
+ if [ $? = 0 ];then
+ success_count=$(( success_count + 1 ))
+ echo "Master brought up successfully."
+ break;
+ fi
+ echo "[check] $i times"
+ sleep 2
+done
+
+# now check if standby1 has successfully joined connected to the master.
+echo "Waiting for the standby1 to join cluster..."
+for i in 1 2 3 4 5 6 7 8 9 10
+do
+ grep "successfully joined the watchdog cluster as standby node" $STANDBY_DIR/log/pgpool.log > /dev/null 2>&1
+ if [ $? = 0 ];then
+ success_count=$(( success_count + 1 ))
+ echo "Standby successfully connected."
+ break;
+ fi
+ echo "[check] $i times"
+ sleep 2
+done
+
+# now check if standby2 has successfully joined connected to the master.
+echo "Waiting for the standby2 to join cluster..."
+for i in 1 2 3 4 5 6 7 8 9 10
+do
+ grep "successfully joined the watchdog cluster as standby node" $STANDBY2_DIR/log/pgpool.log > /dev/null 2>&1
+ if [ $? = 0 ];then
+ success_count=$(( success_count + 1 ))
+ echo "Standby2 successfully connected."
+ break;
+ fi
+ echo "[check] $i times"
+ sleep 2
+done
+
+# raise an artificial communication error on standby for DB node 1
+echo "1 down" > $STANDBY_DIR/log/backend_down_request
+echo "Checking if the standby successfuly process the failover request..."
+for i in 1 2 3 4 5 6 7 8 9 10
+do
+ grep -i "building consensus for request" $STANDBY_DIR/log/pgpool.log
+ if [ $? = 0 ];then
+ success_count=$(( success_count + 1 ))
+ echo "Fake DB error generated and master is waiting for consensus"
+ break;
+ fi
+ echo "[check] $i times"
+ sleep 2
+done
+echo "Checking if the Master receives the failover request and waiting for consensus..."
+for i in 1 2 3 4 5 6 7 8 9 10
+do
+ grep -i "failover request noted" $MASTER_DIR/log/pgpool.log
+ if [ $? = 0 ];then
+ success_count=$(( success_count + 1 ))
+ echo "Fake DB error delivered to master. and master is waiting for consensus"
+ break;
+ fi
+ echo "[check] $i times"
+ sleep 2
+done
+
+# raise an artificial communication error again on the standby for DB node 1
+echo "1 down" > $STANDBY_DIR/log/backend_down_request
+# standby should quarantine the node now
+echo "Checking if the node is quarantined on standby..."
+for i in 1 2 3 4 5 6 7 8 9 10
+do
+ grep -i "changed to quarantine node request" $STANDBY_DIR/log/pgpool.log
+ if [ $? = 0 ];then
+ success_count=$(( success_count + 1 ))
+ echo "Node is quarantined because of second error but no consensus"
+ break;
+ fi
+ echo "[check] $i times"
+ sleep 2
+done
+
+
+# raise an artificial communication on second standby to check if failover is executed this time
+echo "1 down" > $STANDBY2_DIR/log/backend_down_request
+
+#give some time to pgpool-II to execute failover
+sleep 5
+# check to see if all Pgpool-II agrees that the failover request is
+# executed
+echo "Checking if all Pgpool-II agrees that the failover request is executed"
+for i in 1 2 3 4 5 6 7 8 9 10
+do
+ n=0
+ for p in 11000 11100 11200
+ do
+ $PSQL -p $p -c "show pool_nodes" test|grep standby|grep down >/dev/null 2>&1
+ if [ $? = 0 ];then
+ n=$(( n + 1))
+ fi
+ done
+ if [ $n -eq 3 ];then
+ success_count=$(( success_count + 1 ))
+ echo "All Pgpool-II agrees that the failover request is executed"
+ break;
+ fi
+ echo "[check] $i times"
+ sleep 2
+done
+
+
+# we are done. Just stop the standby pgpool-II
+$PGPOOL_INSTALL_DIR/bin/pgpool -f $STANDBY_DIR/etc/pgpool.conf -m f stop
+$PGPOOL_INSTALL_DIR/bin/pgpool -f $STANDBY2_DIR/etc/pgpool.conf -m f stop
+cd master
+./shutdownall
+
+echo "$success_count out of $num_tests successfull";
+
+if test $success_count -eq $num_tests
+then
+ exit 0
+fi
+
+exit 1
diff --git a/src/test/regression/tests/014.watchdoc_test_quorum_bypass/master.conf b/src/test/regression/tests/014.watchdoc_test_quorum_bypass/master.conf
new file mode 100644
index 000000000..58e6f4bad
--- /dev/null
+++ b/src/test/regression/tests/014.watchdoc_test_quorum_bypass/master.conf
@@ -0,0 +1,24 @@
+# master watchdog
+num_init_children = 4
+use_watchdog = on
+failover_when_quorum_exists = true
+failover_require_consensus = true
+allow_multiple_failover_requests_from_node = false
+wd_interval = 1
+wd_priority = 5
+wd_hostname = 'localhost'
+wd_port = 21004
+wd_heartbeat_port = 21005
+
+heartbeat_destination0 = 'localhost'
+heartbeat_destination_port0 = 21105
+heartbeat_destination1 = 'localhost'
+heartbeat_destination_port1 = 21205
+
+other_pgpool_hostname0 = 'localhost'
+other_pgpool_port0 = 11100
+other_wd_port0 = 21104
+
+other_pgpool_hostname1 = 'localhost'
+other_pgpool_port1 = 11200
+other_wd_port1 = 21204
diff --git a/src/test/regression/tests/014.watchdoc_test_quorum_bypass/test.sh b/src/test/regression/tests/014.watchdoc_test_quorum_bypass/test.sh
new file mode 100755
index 000000000..ac906e4e2
--- /dev/null
+++ b/src/test/regression/tests/014.watchdoc_test_quorum_bypass/test.sh
@@ -0,0 +1,81 @@
+#!/usr/bin/env bash
+#-------------------------------------------------------------------
+# test script for watchdog
+#
+# Please note that to successfully run the test, "HEALTHCHECK_DEBUG"
+# must be defined before compiling main/health_check.c.
+#
+# test pcp_detach bypass failover_when_quorum_exists and failover_require_consensus
+#
+source $TESTLIBS
+MASTER_DIR=master
+num_tests=2
+success_count=0
+PSQL=$PGBIN/psql
+PG_CTL=$PGBIN/pg_ctl
+
+rm -fr $MASTER_DIR
+rm -fr $STANDBY_DIR
+rm -fr $STANDBY2_DIR
+
+mkdir $MASTER_DIR
+mkdir $STANDBY_DIR
+mkdir $STANDBY2_DIR
+
+
+# dir in master directory
+cd $MASTER_DIR
+
+# create master environment
+echo -n "creating master pgpool and PostgreSQL clusters..."
+$PGPOOL_SETUP -m s -n 2 -p 11000|| exit 1
+echo "master setup done."
+
+
+source ./bashrc.ports
+cat ../master.conf >> etc/pgpool.conf
+
+./startall
+wait_for_pgpool_startup
+
+
+# back to test root dir
+cd ..
+
+
+# First test check if pgpool-II became a master.
+echo "Waiting for the pgpool master..."
+for i in 1 2 3 4 5 6 7 8 9 10
+do
+ grep "I am the cluster leader node" $MASTER_DIR/log/pgpool.log > /dev/null 2>&1
+ if [ $? = 0 ];then
+ success_count=$(( success_count + 1 ))
+ echo "Master brought up successfully."
+ break;
+ fi
+ echo "[check] $i times"
+ sleep 2
+done
+
+#export PCPPASSFILE=/home/usama/work/community/pgpool2/src/test/regression/tests/067.bug231/testdir/pcppass
+$PGPOOL_INSTALL_DIR/bin/pcp_detach_node -w -h localhost -p $PCP_PORT 1 2>&1
+
+$PSQL -p 11000 -c "show pool_nodes" 2>&1
+
+$PSQL -p 11000 -c "show pool_nodes" test|grep standby|grep down >/dev/null 2>&1
+if [ $? = 0 ];then
+ echo "Failover was successfuly executed"
+ success_count=$(( success_count + 1 ))
+fi
+
+cd master
+./shutdownall
+
+echo "$success_count out of $num_tests successfull";
+
+if test $success_count -eq $num_tests
+then
+ exit 0
+fi
+
+exit 1
diff --git a/src/test/regression/tests/015.watchdoc_test_master_and_backend_fail/master.conf b/src/test/regression/tests/015.watchdoc_test_master_and_backend_fail/master.conf
new file mode 100644
index 000000000..58e6f4bad
--- /dev/null
+++ b/src/test/regression/tests/015.watchdoc_test_master_and_backend_fail/master.conf
@@ -0,0 +1,24 @@
+# master watchdog
+num_init_children = 4
+use_watchdog = on
+failover_when_quorum_exists = true
+failover_require_consensus = true
+allow_multiple_failover_requests_from_node = false
+wd_interval = 1
+wd_priority = 5
+wd_hostname = 'localhost'
+wd_port = 21004
+wd_heartbeat_port = 21005
+
+heartbeat_destination0 = 'localhost'
+heartbeat_destination_port0 = 21105
+heartbeat_destination1 = 'localhost'
+heartbeat_destination_port1 = 21205
+
+other_pgpool_hostname0 = 'localhost'
+other_pgpool_port0 = 11100
+other_wd_port0 = 21104
+
+other_pgpool_hostname1 = 'localhost'
+other_pgpool_port1 = 11200
+other_wd_port1 = 21204
diff --git a/src/test/regression/tests/015.watchdoc_test_master_and_backend_fail/standby.conf b/src/test/regression/tests/015.watchdoc_test_master_and_backend_fail/standby.conf
new file mode 100644
index 000000000..710cbb7ae
--- /dev/null
+++ b/src/test/regression/tests/015.watchdoc_test_master_and_backend_fail/standby.conf
@@ -0,0 +1,26 @@
+# standby watchdog
+num_init_children = 4
+port = 11100
+pcp_port = 11105
+use_watchdog = on
+failover_when_quorum_exists = true
+failover_require_consensus = true
+allow_multiple_failover_requests_from_node = false
+wd_interval = 1
+wd_priority = 3
+wd_hostname = 'localhost'
+wd_port = 21104
+wd_heartbeat_port = 21105
+heartbeat_destination0 = 'localhost'
+heartbeat_destination_port0 = 21005
+heartbeat_destination1 = 'localhost'
+heartbeat_destination_port1 = 21205
+
+other_pgpool_hostname0 = 'localhost'
+other_pgpool_port0 = 11000
+other_wd_port0 = 21004
+
+other_pgpool_hostname1 = 'localhost'
+other_pgpool_port1 = 11200
+other_wd_port1 = 21204
+
diff --git a/src/test/regression/tests/015.watchdoc_test_master_and_backend_fail/standby2.conf b/src/test/regression/tests/015.watchdoc_test_master_and_backend_fail/standby2.conf
new file mode 100644
index 000000000..f44714133
--- /dev/null
+++ b/src/test/regression/tests/015.watchdoc_test_master_and_backend_fail/standby2.conf
@@ -0,0 +1,26 @@
+# standby2 watchdog
+num_init_children = 4
+port = 11200
+pcp_port = 11205
+use_watchdog = on
+failover_when_quorum_exists = true
+failover_require_consensus = true
+allow_multiple_failover_requests_from_node = false
+wd_interval = 1
+wd_priority = 1
+wd_hostname = 'localhost'
+wd_port = 21204
+wd_heartbeat_port = 21205
+heartbeat_destination0 = 'localhost'
+heartbeat_destination_port0 = 21005
+heartbeat_destination1 = 'localhost'
+heartbeat_destination_port1 = 21105
+
+other_pgpool_hostname0 = 'localhost'
+other_pgpool_port0 = 11000
+other_wd_port0 = 21004
+
+other_pgpool_hostname1 = 'localhost'
+other_pgpool_port1 = 11100
+other_wd_port1 = 21104
+
diff --git a/src/test/regression/tests/015.watchdoc_test_master_and_backend_fail/test.sh b/src/test/regression/tests/015.watchdoc_test_master_and_backend_fail/test.sh
new file mode 100755
index 000000000..4aa2f681f
--- /dev/null
+++ b/src/test/regression/tests/015.watchdoc_test_master_and_backend_fail/test.sh
@@ -0,0 +1,186 @@
+#!/usr/bin/env bash
+#-------------------------------------------------------------------
+# test script for watchdog
+#
+# Please note that to successfully run the test, "HEALTHCHECK_DEBUG"
+# must be defined before compiling main/health_check.c.
+#
+# test if master and backend goes down at same time Pgpool-II behaves as expected
+#
+source $TESTLIBS
+MASTER_DIR=master
+STANDBY_DIR=standby
+STANDBY2_DIR=standby2
+num_tests=6
+success_count=0
+PSQL=$PGBIN/psql
+PG_CTL=$PGBIN/pg_ctl
+
+rm -fr $MASTER_DIR
+rm -fr $STANDBY_DIR
+rm -fr $STANDBY2_DIR
+
+mkdir $MASTER_DIR
+mkdir $STANDBY_DIR
+mkdir $STANDBY2_DIR
+
+
+# dir in master directory
+cd $MASTER_DIR
+
+# create master environment
+echo -n "creating master pgpool and PostgreSQL clusters..."
+$PGPOOL_SETUP -m s -n 2 -p 11000|| exit 1
+echo "master setup done."
+
+
+# copy the configurations from master to standby
+cp -r etc ../$STANDBY_DIR/
+
+# copy the configurations from master to standby2
+cp -r etc ../$STANDBY2_DIR/
+
+source ./bashrc.ports
+cat ../master.conf >> etc/pgpool.conf
+
+./startall
+wait_for_pgpool_startup
+
+
+# back to test root dir
+cd ..
+
+
+# create standby environment and start pgpool
+
+mkdir $STANDBY_DIR/log
+echo -n "creating standby pgpool..."
+cat standby.conf >> $STANDBY_DIR/etc/pgpool.conf
+# since we are using the same pgpool-II conf as of master. so change the pid file path in standby pgpool conf
+echo "pid_file_name = '$PWD/pgpool2.pid'" >> $STANDBY_DIR/etc/pgpool.conf
+echo "logdir = $STANDBY_DIR/log" >> $STANDBY_DIR/etc/pgpool.conf
+# start the stnadby pgpool-II by hand
+$PGPOOL_INSTALL_DIR/bin/pgpool -D -n -f $STANDBY_DIR/etc/pgpool.conf -F $STANDBY_DIR/etc/pcp.conf -a $STANDBY_DIR/etc/pool_hba.conf > $STANDBY_DIR/log/pgpool.log 2>&1 &
+
+# create standby2 environment but do not start pgpool
+mkdir $STANDBY2_DIR/log
+echo -n "creating standby2 pgpool..."
+cat standby2.conf >> $STANDBY2_DIR/etc/pgpool.conf
+# since we are using the same pgpool-II conf as of master. so change the pid file path in standby pgpool conf
+echo "pid_file_name = '$PWD/pgpool3.pid'" >> $STANDBY2_DIR/etc/pgpool.conf
+echo "logdir = $STANDBY2_DIR/log" >> $STANDBY2_DIR/etc/pgpool.conf
+# start the stnadby pgpool-II by hand
+$PGPOOL_INSTALL_DIR/bin/pgpool -D -n -f $STANDBY2_DIR/etc/pgpool.conf -F $STANDBY2_DIR/etc/pcp.conf -a $STANDBY2_DIR/etc/pool_hba.conf > $STANDBY2_DIR/log/pgpool.log 2>&1 &
+
+# First test check if both pgpool-II have found their correct place in watchdog cluster.
+echo "Waiting for the pgpool master..."
+for i in 1 2 3 4 5 6 7 8 9 10
+do
+ grep "I am the cluster leader node" $MASTER_DIR/log/pgpool.log > /dev/null 2>&1
+ if [ $? = 0 ];then
+ success_count=$(( success_count + 1 ))
+ echo "Master brought up successfully."
+ break;
+ fi
+ echo "[check] $i times"
+ sleep 2
+done
+
+# now check if standby1 has successfully joined connected to the master.
+echo "Waiting for the standby1 to join cluster..."
+for i in 1 2 3 4 5 6 7 8 9 10
+do
+ grep "successfully joined the watchdog cluster as standby node" $STANDBY_DIR/log/pgpool.log > /dev/null 2>&1
+ if [ $? = 0 ];then
+ success_count=$(( success_count + 1 ))
+ echo "Standby successfully connected."
+ break;
+ fi
+ echo "[check] $i times"
+ sleep 2
+done
+
+# now check if standby2 has successfully joined connected to the master.
+echo "Waiting for the standby2 to join cluster..."
+for i in 1 2 3 4 5 6 7 8 9 10
+do
+ grep "successfully joined the watchdog cluster as standby node" $STANDBY2_DIR/log/pgpool.log > /dev/null 2>&1
+ if [ $? = 0 ];then
+ success_count=$(( success_count + 1 ))
+ echo "Standby2 successfully connected."
+ break;
+ fi
+ echo "[check] $i times"
+ sleep 2
+done
+
+#shutdown master and one PG server by hand
+$PGPOOL_INSTALL_DIR/bin/pgpool -D -n -f $MASTER_DIR/etc/pgpool.conf -m f stop
+$PG_CTL -D $MASTER_DIR/data1 -m f stop
+
+# First test check if both pgpool-II have found their correct place in watchdog cluster.
+echo "Waiting for the standby to become new master..."
+for i in 1 2 3 4 5 6 7 8 9 10
+do
+ grep "I am the cluster leader node" $STANDBY_DIR/log/pgpool.log > /dev/null 2>&1
+ if [ $? = 0 ];then
+ success_count=$(( success_count + 1 ))
+ echo "Standby became new master successfully."
+ break;
+ fi
+ echo "[check] $i times"
+ sleep 2
+done
+#give some time to execute failover
+sleep 5
+
+# First test check if both pgpool-II have found their correct place in watchdog cluster.
+echo "Waiting for the standby to execute failover..."
+for i in 1 2 3 4 5 6 7 8 9 10
+do
+ grep " failover done" $STANDBY_DIR/log/pgpool.log > /dev/null 2>&1
+ if [ $? = 0 ];then
+ success_count=$(( success_count + 1 ))
+ echo "Standby became new master successfully."
+ break;
+ fi
+ echo "[check] $i times"
+ sleep 2
+done
+# check to see if all Pgpool-II agrees that the failover request is
+# executed
+echo "Checking if all Pgpool-II agrees that the failover request was executed"
+for i in 1 2 3 4 5 6 7 8 9 10
+do
+ n=0
+ for p in 11100 11200
+ do
+ $PSQL -p $p -c "show pool_nodes" test|grep standby|grep down >/dev/null 2>&1
+ if [ $? = 0 ];then
+ n=$(( n + 1))
+ fi
+ done
+ if [ $n -eq 2 ];then
+ success_count=$(( success_count + 1 ))
+ echo "All Pgpool-II agrees that the failover request is executed"
+ break;
+ fi
+ echo "[check] $i times"
+ sleep 2
+done
+
+
+# we are done. Just stop the standby pgpool-II
+$PGPOOL_INSTALL_DIR/bin/pgpool -f $STANDBY_DIR/etc/pgpool.conf -m f stop
+$PGPOOL_INSTALL_DIR/bin/pgpool -f $STANDBY2_DIR/etc/pgpool.conf -m f stop
+cd master
+./shutdownall
+
+echo "$success_count out of $num_tests successfull";
+
+if test $success_count -eq $num_tests
+then
+ exit 0
+fi
+
+exit 1
diff --git a/src/watchdog/watchdog.c b/src/watchdog/watchdog.c
index e3a3cfc8d..52bb2cf52 100644
--- a/src/watchdog/watchdog.c
+++ b/src/watchdog/watchdog.c
@@ -3593,7 +3593,7 @@ static int standard_packet_processor(WatchdogNode* wdNode, WDPacketData* pkt)
{
case WD_FAILOVER_WAITING_FOR_CONSENSUS:
ereport(LOG,
- (errmsg("remote node \"%s\" is asking to degenerate quarantined backend node",wdNode->nodeName)));
+ (errmsg("remote node \"%s\" is asking to inform about quarantined backend nodes",wdNode->nodeName)));
register_inform_quarantine_nodes_req();
break;
@@ -3718,7 +3718,7 @@ static int standard_packet_processor(WatchdogNode* wdNode, WDPacketData* pkt)
}
else if (WD_MASTER_NODE == wdNode && oldQuorumStatus != wdNode->quorum_status)
{
- /* inform Pgpool man about quorum status changes */
+ /* inform Pgpool main about quorum status changes */
register_watchdog_quorum_change_interupt();
}
}
[parent: 225ba5e5844d]