pgpool2.git / summary / log / commit / refs
commit 34922b5bbd6bcbb7a94f3a14400415f88db2c3b7
Author: Tatsuo Ishii <ishii@postgresql.org>
Date: Tue Sep 19 01:43:33 2017 +0000
Add new test 011.watchdoc_quorum_failover.
Also modify Makefiles and health_check.c so that only when made with
"HEALTHCHECK_DEBUG=1" option to enable the health_check's "fake
failover" functionality which is necessary for the new test. Always
enabling the functionality is dangerous for a production environment.
src/Makefile.am | 4 +-
src/Makefile.in | 4 +-
src/main/health_check.c | 7 +-
src/test/regression/regress.sh | 2 +-
.../tests/011.watchdoc_quorum_failover/master.conf | 20 ++
.../011.watchdoc_quorum_failover/standby.conf | 22 ++
.../011.watchdoc_quorum_failover/standby2.conf | 22 ++
.../tests/011.watchdoc_quorum_failover/test.sh | 226 +++++++++++++++++++++
8 files changed, 303 insertions(+), 4 deletions(-)
diff --git a/src/Makefile.am b/src/Makefile.am
index f9a8db3e6..e6f04e605 100644
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -2,6 +2,7 @@ AUTOMAKE_OPTIONS = subdir-objects
libs_dir = $(top_srcdir)/src/libs
AM_CPPFLAGS = -D_GNU_SOURCE -I @PGSQL_INCLUDE_DIR@
bin_PROGRAMS = pgpool
+HEALTHCHECK_DEBUG=0
pgpool_SOURCES = main/main.c \
main/pool_globals.c \
@@ -55,7 +56,8 @@ pgpool_SOURCES = main/main.c \
utils/statistics.c
DEFS = @DEFS@ \
- -DDEFAULT_CONFIGDIR=\"$(sysconfdir)\"
+ -DDEFAULT_CONFIGDIR=\"$(sysconfdir)\" \
+ -DHEALTHCHECK_OPTS=$(HEALTHCHECK_DEBUG)
sysconf_DATA = sample/pgpool.conf.sample \
sample/pcp.conf.sample \
diff --git a/src/Makefile.in b/src/Makefile.in
index fac5404c3..ea878651e 100644
--- a/src/Makefile.in
+++ b/src/Makefile.in
@@ -311,7 +311,8 @@ CPP = @CPP@
CPPFLAGS = @CPPFLAGS@
CYGPATH_W = @CYGPATH_W@
DEFS = @DEFS@ \
- -DDEFAULT_CONFIGDIR=\"$(sysconfdir)\"
+ -DDEFAULT_CONFIGDIR=\"$(sysconfdir)\" \
+ -DHEALTHCHECK_OPTS=$(HEALTHCHECK_DEBUG)
DEPDIR = @DEPDIR@
DLLTOOL = @DLLTOOL@
@@ -440,6 +441,7 @@ top_srcdir = @top_srcdir@
AUTOMAKE_OPTIONS = subdir-objects
libs_dir = $(top_srcdir)/src/libs
AM_CPPFLAGS = -D_GNU_SOURCE -I @PGSQL_INCLUDE_DIR@
+HEALTHCHECK_DEBUG = 0
pgpool_SOURCES = main/main.c \
main/pool_globals.c \
main/pgpool_main.c \
diff --git a/src/main/health_check.c b/src/main/health_check.c
index f61585438..e6a8f667d 100644
--- a/src/main/health_check.c
+++ b/src/main/health_check.c
@@ -72,7 +72,12 @@ static RETSIGTYPE reload_config_handler(int sig);
static void reload_config(void);
static RETSIGTYPE health_check_timer_handler(int sig);
-#undef HEALTHCHECK_DEBUG
+#ifdef HEALTHCHECK_OPTS
+ #if HEALTHCHECK_OPTS > 0
+ #define HEALTHCHECK_DEBUG
+ #endif
+#endif
+
#ifdef HEALTHCHECK_DEBUG
static bool check_backend_down_request(int node);
#endif
diff --git a/src/test/regression/regress.sh b/src/test/regression/regress.sh
index d89ae97bb..a70d8dae5 100755
--- a/src/test/regression/regress.sh
+++ b/src/test/regression/regress.sh
@@ -37,7 +37,7 @@ function install_pgpool
test -d $log || mkdir $log
- make install -C $dir/../../ -e prefix=${PGPOOL_PATH} >& $log/regression.log 2>&1
+ make install HEALTHCHECK_DEBUG=1 -C $dir/../../ -e prefix=${PGPOOL_PATH} >& regression.log 2>&1
if [ $? != 0 ];then
echo "make install failed"
diff --git a/src/test/regression/tests/011.watchdoc_quorum_failover/master.conf b/src/test/regression/tests/011.watchdoc_quorum_failover/master.conf
new file mode 100644
index 000000000..289ca5afa
--- /dev/null
+++ b/src/test/regression/tests/011.watchdoc_quorum_failover/master.conf
@@ -0,0 +1,20 @@
+# master watchdog
+use_watchdog = on
+wd_interval = 1
+wd_priority = 5
+wd_hostname = 'localhost'
+wd_port = 21004
+wd_heartbeat_port = 21005
+
+heartbeat_destination0 = 'localhost'
+heartbeat_destination_port0 = 21105
+heartbeat_destination1 = 'localhost'
+heartbeat_destination_port1 = 21205
+
+other_pgpool_hostname0 = 'localhost'
+other_pgpool_port0 = 11100
+other_wd_port0 = 21104
+
+other_pgpool_hostname1 = 'localhost'
+other_pgpool_port1 = 11200
+other_wd_port1 = 21204
diff --git a/src/test/regression/tests/011.watchdoc_quorum_failover/standby.conf b/src/test/regression/tests/011.watchdoc_quorum_failover/standby.conf
new file mode 100644
index 000000000..120bcb0f8
--- /dev/null
+++ b/src/test/regression/tests/011.watchdoc_quorum_failover/standby.conf
@@ -0,0 +1,22 @@
+# standby watchdog
+port = 11100
+pcp_port = 11105
+use_watchdog = on
+wd_interval = 1
+wd_priority = 3
+wd_hostname = 'localhost'
+wd_port = 21104
+wd_heartbeat_port = 21105
+heartbeat_destination0 = 'localhost'
+heartbeat_destination_port0 = 21005
+heartbeat_destination1 = 'localhost'
+heartbeat_destination_port1 = 21205
+
+other_pgpool_hostname0 = 'localhost'
+other_pgpool_port0 = 11000
+other_wd_port0 = 21004
+
+other_pgpool_hostname1 = 'localhost'
+other_pgpool_port1 = 11200
+other_wd_port1 = 21204
+
diff --git a/src/test/regression/tests/011.watchdoc_quorum_failover/standby2.conf b/src/test/regression/tests/011.watchdoc_quorum_failover/standby2.conf
new file mode 100644
index 000000000..d5da7ffda
--- /dev/null
+++ b/src/test/regression/tests/011.watchdoc_quorum_failover/standby2.conf
@@ -0,0 +1,22 @@
+# standby2 watchdog
+port = 11200
+pcp_port = 11205
+use_watchdog = on
+wd_interval = 1
+wd_priority = 1
+wd_hostname = 'localhost'
+wd_port = 21204
+wd_heartbeat_port = 21205
+heartbeat_destination0 = 'localhost'
+heartbeat_destination_port0 = 21005
+heartbeat_destination1 = 'localhost'
+heartbeat_destination_port1 = 21105
+
+other_pgpool_hostname0 = 'localhost'
+other_pgpool_port0 = 11000
+other_wd_port0 = 21004
+
+other_pgpool_hostname1 = 'localhost'
+other_pgpool_port1 = 11100
+other_wd_port1 = 21104
+
diff --git a/src/test/regression/tests/011.watchdoc_quorum_failover/test.sh b/src/test/regression/tests/011.watchdoc_quorum_failover/test.sh
new file mode 100755
index 000000000..ddfeed926
--- /dev/null
+++ b/src/test/regression/tests/011.watchdoc_quorum_failover/test.sh
@@ -0,0 +1,226 @@
+#!/usr/bin/env bash
+#-------------------------------------------------------------------
+# test script for watchdog
+#
+# Please note that to successfully run the test, "HEALTHCHECK_DEBUG"
+# must be defined before compiling main/health_check.c.
+
+source $TESTLIBS
+MASTER_DIR=master
+STANDBY_DIR=standby
+STANDBY2_DIR=standby2
+num_tests=9
+success_count=0
+
+rm -fr $MASTER_DIR
+rm -fr $STANDBY_DIR
+rm -fr $STANDBY2_DIR
+
+mkdir $MASTER_DIR
+mkdir $STANDBY_DIR
+mkdir $STANDBY2_DIR
+
+
+# dir in master directory
+cd $MASTER_DIR
+
+# create master environment
+echo -n "creating master pgpool and PostgreSQL clusters..."
+$PGPOOL_SETUP -m s -n 2 -p 11000|| exit 1
+echo "master setup done."
+
+
+# copy the configurations from master to standby
+cp -r etc ../$STANDBY_DIR/
+
+# copy the configurations from master to standby2
+cp -r etc ../$STANDBY2_DIR/
+
+source ./bashrc.ports
+cat ../master.conf >> etc/pgpool.conf
+
+./startall
+wait_for_pgpool_startup
+
+
+# back to test root dir
+cd ..
+
+
+# create standby environment
+
+mkdir $STANDBY_DIR/log
+echo -n "creating standby pgpool..."
+cat standby.conf >> $STANDBY_DIR/etc/pgpool.conf
+# since we are using the same pgpool-II conf as of master. so change the pid file path in standby pgpool conf
+echo "pid_file_name = '$PWD/pgpool2.pid'" >> $STANDBY_DIR/etc/pgpool.conf
+echo "logdir = $STANDBY_DIR/log" >> $STANDBY_DIR/etc/pgpool.conf
+# start the stnadby pgpool-II by hand
+$PGPOOL_INSTALL_DIR/bin/pgpool -D -n -f $STANDBY_DIR/etc/pgpool.conf -F $STANDBY_DIR/etc/pcp.conf -a $STANDBY_DIR/etc/pool_hba.conf > $STANDBY_DIR/log/pgpool.log 2>&1 &
+
+# create standby2 environment
+
+mkdir $STANDBY2_DIR/log
+echo -n "creating standby2 pgpool..."
+cat standby2.conf >> $STANDBY2_DIR/etc/pgpool.conf
+# since we are using the same pgpool-II conf as of master. so change the pid file path in standby pgpool conf
+echo "pid_file_name = '$PWD/pgpool3.pid'" >> $STANDBY2_DIR/etc/pgpool.conf
+echo "logdir = $STANDBY2_DIR/log" >> $STANDBY2_DIR/etc/pgpool.conf
+# start the stnadby pgpool-II by hand
+$PGPOOL_INSTALL_DIR/bin/pgpool -D -n -f $STANDBY2_DIR/etc/pgpool.conf -F $STANDBY2_DIR/etc/pcp.conf -a $STANDBY2_DIR/etc/pool_hba.conf > $STANDBY2_DIR/log/pgpool.log 2>&1 &
+
+# First test check if both pgpool-II have found their correct place in watchdog cluster.
+echo "Waiting for the pgpool master..."
+for i in 1 2 3 4 5 6 7 8 9 10
+do
+ grep "I am the cluster leader node" $MASTER_DIR/log/pgpool.log > /dev/null 2>&1
+ if [ $? = 0 ];then
+ success_count=$(( success_count + 1 ))
+ echo "Master brought up successfully."
+ break;
+ fi
+ echo "[check] $i times"
+ sleep 2
+done
+
+# now check if standby has successfully joined connected to the master.
+echo "Waiting for the standby to join cluster..."
+for i in 1 2 3 4 5 6 7 8 9 10
+do
+ grep "successfully joined the watchdog cluster as standby node" $STANDBY_DIR/log/pgpool.log > /dev/null 2>&1
+ if [ $? = 0 ];then
+ success_count=$(( success_count + 1 ))
+ echo "Standby2 successfully connected."
+ break;
+ fi
+ echo "[check] $i times"
+ sleep 2
+done
+
+# now check if standby2 has successfully joined connected to the master.
+echo "Waiting for the standby2 to join cluster..."
+for i in 1 2 3 4 5 6 7 8 9 10
+do
+ grep "successfully joined the watchdog cluster as standby node" $STANDBY2_DIR/log/pgpool.log > /dev/null 2>&1
+ if [ $? = 0 ];then
+ success_count=$(( success_count + 1 ))
+ echo "Standby2 successfully connected."
+ break;
+ fi
+ echo "[check] $i times"
+ sleep 2
+done
+
+# raise an artificial communication error on standby2 for DB node 1
+echo "1 down" > $STANDBY2_DIR/log/backend_down_request
+echo "Checking if the Standby2 fooled by fake error..."
+for i in 1 2 3 4 5 6 7 8 9 10
+do
+ grep -i "failover" $STANDBY2_DIR/log/pgpool.log
+ if [ $? = 0 ];then
+ success_count=$(( success_count + 1 ))
+ echo "Fake DB error detected."
+ break;
+ fi
+ echo "[check] $i times"
+ sleep 2
+done
+
+# check to see if all Pgpool-II agrees that the failover request is
+# discarded.
+echo "Checking if all Pgpool-II agrees that the failover request is discarded."
+n=0
+for p in 11000 11100 11200
+do
+ psql -p $p -c "show pool_nodes" test|grep standby|grep up >/dev/null 2>&1
+ if [ $? = 0 ];then
+ n=$(( n + 1))
+ fi
+done
+if [ $n -eq 3 ];then
+ echo "All Pgpool-II agrees that the failover request is discarded."
+ success_count=$(( success_count + 1 ))
+fi
+
+# raise an real DB node 1 error
+pg_ctl -D master/data1 -m f stop
+echo "Checking if master detects the shutdown error"
+for i in 1 2 3 4 5 6 7 8 9 10
+do
+ grep -i "failover" $MASTER_DIR/log/pgpool.log
+ if [ $? = 0 ];then
+ success_count=$(( success_count + 1 ))
+ echo "DB error detected."
+ break;
+ fi
+ echo "[check] $i times"
+ sleep 2
+done
+
+# check to see if all Pgpool-II agrees that the failover request is
+# executed
+echo "Checking if all Pgpool-II agrees that the failover request is executed"
+for i in 1 2 3 4 5 6 7 8 9 10
+do
+ n=0
+ for p in 11000 11100 11200
+ do
+ psql -p $p -c "show pool_nodes" test|grep standby|grep down >/dev/null 2>&1
+ if [ $? = 0 ];then
+ n=$(( n + 1))
+ fi
+ done
+ if [ $n -eq 3 ];then
+ success_count=$(( success_count + 1 ))
+ echo "All Pgpool-II agrees that the failover request is executed"
+ break;
+ fi
+ echo "[check] $i times"
+ sleep 2
+done
+
+# stop master pgpool and see if standby takes over the roll
+$PGPOOL_INSTALL_DIR/bin/pgpool -f $MASTER_DIR/etc/pgpool.conf -m f stop
+
+echo "Checking if the Standby pgpool-II detected the master shutdown..."
+for i in 1 2 3 4 5 6 7 8 9 10
+do
+ grep " is shutting down" $STANDBY_DIR/log/pgpool.log > /dev/null 2>&1
+ if [ $? = 0 ];then
+ success_count=$(( success_count + 1 ))
+ echo "Master shutdown detected."
+ break;
+ fi
+ echo "[check] $i times"
+ sleep 2
+done
+
+# Finally see if standby take over
+
+echo "Checking if the Standby pgpool-II takes over the master responsibility..."
+for i in 1 2 3 4 5 6 7 8 9 10
+do
+ grep "I am the cluster leader node" $STANDBY_DIR/log/pgpool.log > /dev/null 2>&1
+ if [ $? = 0 ];then
+ success_count=$(( success_count + 1 ))
+ echo "Standby successfully became the new master."
+ break;
+ fi
+ echo "[check] $i times"
+ sleep 2
+done
+
+# we are done. Just stop the standby pgpool-II
+$PGPOOL_INSTALL_DIR/bin/pgpool -f $STANDBY_DIR/etc/pgpool.conf -m f stop
+$PGPOOL_INSTALL_DIR/bin/pgpool -f $STANDBY2_DIR/etc/pgpool.conf -m f stop
+cd master
+./shutdownall
+
+echo "$success_count out of $num_tests successfull";
+
+if test $success_count -eq $num_tests
+then
+ exit 0
+fi
+
+exit 1
[parent: 54af632c6360]