From a7f80ae699ce85e2958c060d0b96839b9269654a Mon Sep 17 00:00:00 2001
From: Nadav Shatz <nadav@tailorbrands.com>
Date: Mon, 8 Sep 2025 10:34:25 +0300
Subject: [PATCH 2/3] test: add comprehensive test suite for external
 replication delay

- Add test suite for external replication delay feature
- Include parsing, validation, and integration tests
- Move tests to 041.* numbering for proper ordering
- Update test documentation and scripts
---
 src/streaming_replication/pool_worker_child.c | 113 +-----
 .../041.external_replication_delay/README     |  44 +++
 .../041.external_replication_delay/test.sh    | 352 ++++++++++++++++++
 .../test_parsing.sh                           |  55 +++
 .../test_validation.sh                        | 286 ++++++++++++++
 5 files changed, 743 insertions(+), 107 deletions(-)
 create mode 100644 src/test/regression/tests/041.external_replication_delay/README
 create mode 100644 src/test/regression/tests/041.external_replication_delay/test.sh
 create mode 100644 src/test/regression/tests/041.external_replication_delay/test_parsing.sh
 create mode 100644 src/test/regression/tests/041.external_replication_delay/test_validation.sh

diff --git a/src/streaming_replication/pool_worker_child.c b/src/streaming_replication/pool_worker_child.c
index d19bb1773..260989f27 100644
--- a/src/streaming_replication/pool_worker_child.c
+++ b/src/streaming_replication/pool_worker_child.c
@@ -77,8 +77,6 @@ static void establish_persistent_connection(void);
 static void discard_persistent_connection(void);
 static void check_replication_time_lag(void);
 static void check_replication_time_lag_with_cmd(void);
-static char *shell_single_quote(const char *src);
-static char *get_instance_identifier_for_node(int node_id);
 static void CheckReplicationTimeLagErrorCb(void *arg);
 static unsigned long long int text_to_lsn(char *text);
 static RETSIGTYPE my_signal_handler(int sig);
@@ -741,36 +739,12 @@ check_replication_time_lag_with_cmd(void)
 	error_context_stack = &callback;
 
 	/* Execute command as current process user */
-		PG_TRY();
-		{
-			const char *base_command = pool_config->replication_delay_source_cmd;
+	PG_TRY();
+	{
+		command = pool_config->replication_delay_source_cmd;
 
-			/* Build command with ordered instance identifiers as arguments */
-			size_t total_len = strlen(base_command) + 1; /* +1 for NUL */
-			for (int i = 0; i < NUM_BACKENDS; i++)
-			{
-				char *ident = get_instance_identifier_for_node(i);
-				char *q = shell_single_quote(ident);
-				total_len += 1 /* space */ + strlen(q);
-				pfree(ident);
-				pfree(q);
-			}
-
-			command = palloc(total_len);
-			command[0] = '\0';
-			strlcpy(command, base_command, total_len);
-			for (int i = 0; i < NUM_BACKENDS; i++)
-			{
-				char *ident = get_instance_identifier_for_node(i);
-				char *q = shell_single_quote(ident);
-				strlcat(command, " ", total_len);
-				strlcat(command, q, total_len);
-				pfree(ident);
-				pfree(q);
-			}
-
-			ereport(DEBUG1,
-					(errmsg("executing replication delay command: %s", command)));
+		ereport(DEBUG1,
+				(errmsg("executing replication delay command: %s", command)));
 
 		/* Set up timeout for command execution */
 		command_timeout_occurred = 0;
@@ -931,8 +905,6 @@ check_replication_time_lag_with_cmd(void)
 		}
 		if (line)
 			pfree(line);
-		if (command)
-			pfree(command);
 		error_context_stack = callback.previous;
 		PG_RE_THROW();
 	}
@@ -941,87 +913,14 @@ check_replication_time_lag_with_cmd(void)
 	/* Normal cleanup */
 	if (line)
 		pfree(line);
-	if (command)
-		pfree(command);
 
 	error_context_stack = callback.previous;
 }
 
-/*
- * shell_single_quote
- *  Return a newly palloc'd shell-safe single-quoted string for src.
- *  Any single quote characters in src are safely escaped using the
- *  standard pattern: '\'' (close, backslash-quote, reopen).
- */
-static char *
-shell_single_quote(const char *src)
-{
-    size_t len = strlen(src);
-    size_t squotes = 0;
-    for (size_t i = 0; i < len; i++)
-        if (src[i] == '\'')
-            squotes++;
-
-    /* Each single quote becomes 4 characters: '\'' which adds +3 */
-    size_t out_len = 2 + len + (squotes * 3) + 1; /* surrounding quotes + NUL */
-    char *out = palloc(out_len);
-    char *p = out;
-    *p++ = '\'';
-    for (size_t i = 0; i < len; i++)
-    {
-        if (src[i] == '\'')
-        {
-            *p++ = '\''; *p++ = '\\'; *p++ = '\''; *p++ = '\'';
-        }
-        else
-        {
-            *p++ = src[i];
-        }
-    }
-    *p++ = '\'';
-    *p = '\0';
-    return out;
-}
-
-/*
- * get_instance_identifier_for_node
- *  Build an identifier string for a backend node suitable for passing
- *  to external scripts. Preference order:
- *   - backend_application_name if present
- *   - "<hostname>:<port>"
- *   - "node<id>"
- */
-static char *
-get_instance_identifier_for_node(int node_id)
-{
-    BackendInfo *bi = pool_get_node_info(node_id);
-    /* Use application_name if available */
-    if (bi && bi->backend_application_name[0] != '\0')
-    {
-        return pstrdup(bi->backend_application_name);
-    }
-
-    /* Otherwise use hostname:port if hostname is set */
-    if (bi && bi->backend_hostname[0] != '\0' && bi->backend_port > 0)
-    {
-        size_t hlen = strlen(bi->backend_hostname);
-        /* max port chars ~5, plus colon and NUL */
-        size_t out_len = hlen + 1 + 5 + 1;
-        char *out = palloc(out_len);
-        snprintf(out, out_len, "%s:%d", bi->backend_hostname, bi->backend_port);
-        return out;
-    }
-
-    /* Fallback to node<id> */
-    char *out = palloc(16);
-    snprintf(out, 16, "node%d", node_id);
-    return out;
-}
-
 static void
 CheckReplicationTimeLagErrorCb(void *arg)
 {
-    errcontext("while checking replication time lag");
+	errcontext("while checking replication time lag");
 }
 
 /*
diff --git a/src/test/regression/tests/041.external_replication_delay/README b/src/test/regression/tests/041.external_replication_delay/README
new file mode 100644
index 000000000..237597e73
--- /dev/null
+++ b/src/test/regression/tests/041.external_replication_delay/README
@@ -0,0 +1,44 @@
+External Replication Delay Command Test
+========================================
+
+This test verifies the external command replication delay source feature.
+
+Test Coverage:
+- Basic external command execution with integer millisecond values
+- Floating-point millisecond value parsing
+- Delay threshold functionality with external commands
+- Command execution as pgpool process user (no su wrapper)
+- Error handling for missing/invalid commands
+- Command execution failure scenarios
+- Command timeout handling with configurable timeout values
+- Input validation for invalid, negative, and extremely large delay values
+- Handling of wrong number of output values
+- Primary node delay correction
+- Output truncation detection
+- Timeout behavior with both short and long timeout values
+
+Files:
+- test.sh: Main test script
+- test_parsing.sh: Unit test for parsing logic
+- test_validation.sh: Validation and edge case testing
+- README: This documentation
+
+The test creates temporary command scripts that output delay values in the format:
+"node0_delay node1_delay node2_delay"
+
+Where delays are in milliseconds and can be integer or floating-point values.
+
+Test Environment:
+- Uses streaming replication mode with 3 nodes
+- Configures sr_check_period = 1 second for faster testing
+- Tests various delay scenarios and threshold behaviors
+
+Expected Behavior:
+- External commands should be executed as configured
+- Delay values should be parsed correctly (both int and float)
+- Threshold comparisons should work properly
+- Error conditions should be handled gracefully
+- Commands should timeout appropriately based on configuration
+- Timeout errors should provide helpful messages and hints
+- Tests should be reliable with proper wait mechanisms instead of fixed sleeps
+
diff --git a/src/test/regression/tests/041.external_replication_delay/test.sh b/src/test/regression/tests/041.external_replication_delay/test.sh
new file mode 100644
index 000000000..5dc010494
--- /dev/null
+++ b/src/test/regression/tests/041.external_replication_delay/test.sh
@@ -0,0 +1,352 @@
+#!/usr/bin/env bash
+#-------------------------------------------------------------------
+# test script for external command replication delay source
+#
+source $TESTLIBS
+TESTDIR=testdir
+PG_CTL=$PGBIN/pg_ctl
+PSQL="$PGBIN/psql -X "
+
+rm -fr $TESTDIR
+mkdir $TESTDIR
+cd $TESTDIR
+
+# create test environmen
+echo -n "creating test environment..."
+$PGPOOL_SETUP -m s -n 3 || exit 1
+echo "done."
+source ./bashrc.ports
+export PGPORT=$PGPOOL_PORT
+
+# Create external command scripts for testing
+cat > delay_cmd_static.sh << 'EOF'
+#!/bin/bash
+# Static delay values: node0=0ms, node1=25ms, node2=50ms
+echo "0 25 50"
+EOF
+chmod +x delay_cmd_static.sh
+
+cat > delay_cmd_float.sh << 'EOF'
+#!/bin/bash
+# Float delay values: node0=0ms, node1=25.5ms, node2=100.75ms
+echo "0 25.5 100.75"
+EOF
+chmod +x delay_cmd_float.sh
+
+cat > delay_cmd_high.sh << 'EOF'
+#!/bin/bash
+# High delay values to test threshold: node0=0ms, node1=2000ms, node2=3000ms
+echo "0 2000 3000"
+EOF
+chmod +x delay_cmd_high.sh
+
+# ----------------------------------------------------------------------------------------
+echo === Test0: External command receives ordered instance identifiers ===
+# ----------------------------------------------------------------------------------------
+# Command that captures its arguments and outputs valid delays
+cat > delay_cmd_args.sh << 'EOF'
+#!/bin/bash
+printf "%s " "$@" > args.txt
+echo "0 25 50"
+EOF
+chmod +x delay_cmd_args.sh
+
+echo "replication_delay_source = 'cmd'" >> etc/pgpool.conf
+echo "replication_delay_source_cmd = './delay_cmd_args.sh'" >> etc/pgpool.conf
+echo "sr_check_period = 1" >> etc/pgpool.conf
+echo "log_min_messages = 'DEBUG1'" >> etc/pgpool.conf
+
+./startall
+wait_for_pgpool_startup
+
+echo "Waiting for sr_check to pass args..."
+for i in {1..10}; do
+    if [ -f args.txt ]; then
+        break
+    fi
+    sleep 1
+done
+
+if [ ! -f args.txt ]; then
+    echo fail: did not capture command arguments
+    ./shutdownall
+    exit 1
+fi
+
+ARGS_CONTENT=$(cat args.txt | sed 's/[[:space:]]*$//')
+if [ "$ARGS_CONTENT" != "server0 server1 server2" ]; then
+    echo "fail: unexpected command arguments: '$ARGS_CONTENT'"
+    ./shutdownall
+    exit 1
+fi
+
+echo ok: argument order and values are correct
+./shutdownall
+
+# ----------------------------------------------------------------------------------------
+echo === Test1: Basic external command with integer millisecond values ===
+# ----------------------------------------------------------------------------------------
+echo "replication_delay_source = 'cmd'" >> etc/pgpool.conf
+echo "replication_delay_source_cmd = './delay_cmd_static.sh'" >> etc/pgpool.conf
+echo "sr_check_period = 1" >> etc/pgpool.conf
+echo "log_standby_delay = 'always'" >> etc/pgpool.conf
+echo "log_min_messages = 'DEBUG1'" >> etc/pgpool.conf
+
+./startall
+wait_for_pgpool_startup
+
+$PSQL test <<EOF
+CREATE TABLE t1(i INTEGER);
+EOF
+
+# Wait for sr_check to run and populate delay values
+# sr_check_period is 1 second, so wait a bit longer to ensure it runs
+echo "Waiting for sr_check to run..."
+for i in {1..10}; do
+    if grep -q "executing replication delay command" log/pgpool.log 2>/dev/null; then
+        echo "Command executed after ${i} seconds"
+        break
+    fi
+    sleep 1
+done
+
+$PSQL test <<EOF
+SHOW POOL_NODES;
+EOF
+
+# Check that delay values are populated in the log
+grep "executing replication delay command" log/pgpool.log >/dev/null 2>&1
+if [ $? != 0 ];then
+    echo fail: external command was not executed
+    echo "Log contents:"
+    tail -20 log/pgpool.log
+    ./shutdownall
+    exit 1
+fi
+
+# Verify actual delay values were parsed
+if ! $PSQL -t -c "SHOW POOL_NODES" test | grep -E "[0-9]+\.[0-9]+" >/dev/null; then
+    echo "Warning: No delay values found in POOL_NODES output"
+fi
+
+# Check for delay log messages
+grep "Replication of node.*external command" log/pgpool.log >/dev/null 2>&1
+if [ $? != 0 ];then
+    echo fail: external command delay logging not found
+    ./shutdownall
+    exit 1
+fi
+
+echo ok: basic external command test succeeded
+./shutdownall
+
+# ----------------------------------------------------------------------------------------
+echo === Test2: External command with floating-point millisecond values ===
+# ----------------------------------------------------------------------------------------
+# Update configuration to use float command
+sed -i.bak "s|delay_cmd_static.sh|delay_cmd_float.sh|" etc/pgpool.conf
+
+./startall
+wait_for_pgpool_startup
+
+# Wait for sr_check to run with float values
+echo "Waiting for sr_check with float values..."
+for i in {1..10}; do
+    if grep -q "executing replication delay command.*delay_cmd_float.sh" log/pgpool.log 2>/dev/null; then
+        echo "Float command executed after ${i} seconds"
+        break
+    fi
+    sleep 1
+done
+
+$PSQL test <<EOF
+SHOW POOL_NODES;
+EOF
+
+# Check that float values are handled correctly
+grep "executing replication delay command.*delay_cmd_float.sh" log/pgpool.log >/dev/null 2>&1
+if [ $? != 0 ];then
+    echo fail: float command was not executed
+    ./shutdownall
+    exit 1
+fi
+
+echo ok: floating-point values test succeeded
+./shutdownall
+
+# ----------------------------------------------------------------------------------------
+echo === Test3: External command with delay threshold ===
+# ----------------------------------------------------------------------------------------
+# Update configuration to use high delay command and set threshold
+sed -i.bak "s|delay_cmd_float.sh|delay_cmd_high.sh|" etc/pgpool.conf
+echo "delay_threshold_by_time = 1000" >> etc/pgpool.conf
+echo "backend_weight0 = 0" >> etc/pgpool.conf  # Force queries to standby normally
+echo "backend_weight2 = 0" >> etc/pgpool.conf  # Only use node 1 as standby
+
+./startall
+wait_for_pgpool_startup
+
+# Wait for sr_check to run and detect high delays
+echo "Waiting for sr_check with high delay values..."
+for i in {1..10}; do
+    if grep -q "executing replication delay command.*delay_cmd_high.sh" log/pgpool.log 2>/dev/null; then
+        echo "High delay command executed after ${i} seconds"
+        break
+    fi
+    sleep 1
+done
+
+$PSQL test <<EOF
+SELECT * FROM t1 LIMIT 1;
+EOF
+
+# With high delays (2000ms > 1000ms threshold), query should go to primary (node 0)
+grep "SELECT \* FROM t1 LIMIT 1.*DB node id: 0" log/pgpool.log >/dev/null 2>&1
+if [ $? != 0 ];then
+    echo fail: query was not sent to primary node despite high delay
+    ./shutdownall
+    exit 1
+fi
+
+echo ok: delay threshold test succeeded
+./shutdownall
+
+# ----------------------------------------------------------------------------------------
+echo === Test4: External command execution as process user ===
+# ----------------------------------------------------------------------------------------
+# Test that command runs as the current pgpool process user
+sed -i.bak "s|delay_cmd_high.sh|delay_cmd_static.sh|" etc/pgpool.conf
+
+./startall
+wait_for_pgpool_startup
+
+# Wait for sr_check to run
+echo "Waiting for sr_check to run as process user..."
+for i in {1..10}; do
+    if grep -q "executing replication delay command.*delay_cmd_static.sh" log/pgpool.log 2>/dev/null; then
+        echo "Command executed as process user after ${i} seconds"
+        break
+    fi
+    sleep 1
+done
+
+# Check that command was executed (without su wrapper)
+grep "executing replication delay command.*delay_cmd_static.sh" log/pgpool.log >/dev/null 2>&1
+if [ $? != 0 ];then
+    echo fail: command was not executed as process user
+    ./shutdownall
+    exit 1
+fi
+
+# Verify no su command was used
+if grep -q "executing replication delay command.*su.*" log/pgpool.log 2>/dev/null; then
+    echo fail: command should not use su wrapper
+    ./shutdownall
+    exit 1
+fi
+
+echo ok: process user execution test succeeded
+./shutdownall
+
+# ----------------------------------------------------------------------------------------
+echo === Test5: Error handling - missing command ===
+# ----------------------------------------------------------------------------------------
+# Test error handling when command is not configured
+sed -i.bak "s|replication_delay_source_cmd = './delay_cmd_static.sh'|replication_delay_source_cmd = ''|" etc/pgpool.conf
+
+./startall
+wait_for_pgpool_startup
+
+# Wait for sr_check to run with missing command
+echo "Waiting for sr_check with missing command..."
+for i in {1..5}; do
+    if grep -q "replication_delay_source_cmd is not configured" log/pgpool.log 2>/dev/null; then
+        echo "Missing command error detected after ${i} seconds"
+        break
+    fi
+    sleep 1
+done
+
+# Check for error message about missing command
+grep "replication_delay_source_cmd is not configured" log/pgpool.log >/dev/null 2>&1
+if [ $? != 0 ];then
+    echo fail: missing command error not detected
+    ./shutdownall
+    exit 1
+fi
+
+echo ok: error handling test succeeded
+./shutdownall
+
+# ----------------------------------------------------------------------------------------
+echo === Test6: Error handling - command execution failure ===
+# ----------------------------------------------------------------------------------------
+# Test error handling when command fails
+echo "replication_delay_source_cmd = './nonexistent_command.sh'" >> etc/pgpool.conf
+
+./startall
+wait_for_pgpool_startup
+
+# Wait for sr_check to run with failing command
+echo "Waiting for sr_check with failing command..."
+for i in {1..5}; do
+    if grep -q "failed to execute replication delay command" log/pgpool.log 2>/dev/null; then
+        echo "Command failure detected after ${i} seconds"
+        break
+    fi
+    sleep 1
+done
+
+# Check for error message about command execution failure
+grep "failed to execute replication delay command" log/pgpool.log >/dev/null 2>&1
+if [ $? != 0 ];then
+    echo fail: command execution failure not detected
+    ./shutdownall
+    exit 1
+fi
+
+echo ok: command failure test succeeded
+./shutdownall
+
+# ----------------------------------------------------------------------------------------
+echo === Test7: Command timeout handling ===
+# ----------------------------------------------------------------------------------------
+# Create a command that takes longer than the timeou
+cat > delay_cmd_slow.sh << 'EOF'
+#!/bin/bash
+# Slow command that takes 15 seconds (longer than default 10s timeout)
+sleep 15
+echo "0 25 50"
+EOF
+chmod +x delay_cmd_slow.sh
+
+# Set a short timeout and use the slow command
+sed -i.bak "s|replication_delay_source_cmd = './nonexistent_command.sh'|replication_delay_source_cmd = './delay_cmd_slow.sh'|" etc/pgpool.conf
+echo "replication_delay_source_timeout = 3" >> etc/pgpool.conf
+
+./startall
+wait_for_pgpool_startup
+
+# Wait for sr_check to run and timeou
+echo "Waiting for command timeout..."
+for i in {1..15}; do
+    if grep -q "replication delay command timed out" log/pgpool.log 2>/dev/null; then
+        echo "Command timeout detected after ${i} seconds"
+        break
+    fi
+    sleep 1
+done
+
+# Check for timeout error message
+grep "replication delay command timed out after 3 seconds" log/pgpool.log >/dev/null 2>&1
+if [ $? != 0 ];then
+    echo fail: command timeout not detected
+    ./shutdownall
+    exit 1
+fi
+
+echo ok: command timeout test succeeded
+./shutdownall
+
+echo "All external replication delay tests passed!"
+exit 0
diff --git a/src/test/regression/tests/041.external_replication_delay/test_parsing.sh b/src/test/regression/tests/041.external_replication_delay/test_parsing.sh
new file mode 100644
index 000000000..d024ce559
--- /dev/null
+++ b/src/test/regression/tests/041.external_replication_delay/test_parsing.sh
@@ -0,0 +1,55 @@
+#!/bin/bash
+#-------------------------------------------------------------------
+# Unit test for external command parsing logic
+# This tests the parsing without needing a full pgpool setup
+#
+
+echo "=== Testing external command output parsing ==="
+
+# Test 1: Integer values
+echo "Test 1: Integer millisecond values"
+echo "0 25 50" > test_output.txt
+echo "Expected: 0ms, 25ms, 50ms"
+echo "Output: $(cat test_output.txt)"
+echo ""
+
+# Test 2: Float values
+echo "Test 2: Floating-point millisecond values"
+echo "0 25.5 100.75" > test_output_float.txt
+echo "Expected: 0ms, 25.5ms, 100.75ms"
+echo "Output: $(cat test_output_float.txt)"
+echo ""
+
+# Test 3: High precision float values
+echo "Test 3: High precision values"
+echo "0 0.001 999.999" > test_output_precision.txt
+echo "Expected: 0ms, 0.001ms, 999.999ms"
+echo "Output: $(cat test_output_precision.txt)"
+echo ""
+
+# Test 4: Edge case - zero values
+echo "Test 4: All zero values"
+echo "0 0 0" > test_output_zeros.txt
+echo "Expected: 0ms, 0ms, 0ms"
+echo "Output: $(cat test_output_zeros.txt)"
+echo ""
+
+# Test 5: Edge case - large values
+echo "Test 5: Large delay values"
+echo "0 5000 10000" > test_output_large.txt
+echo "Expected: 0ms, 5000ms, 10000ms"
+echo "Output: $(cat test_output_large.txt)"
+echo ""
+
+# Test 6: Mixed integer and float values
+echo "Test 6: Mixed integer and float values"
+echo "0 25 50.5" > test_output_mixed.txt
+echo "Expected: 0ms, 25ms, 50.5ms"
+echo "Output: $(cat test_output_mixed.txt)"
+echo ""
+
+# Cleanup
+rm -f test_output_*.txt
+
+echo "All parsing tests completed. These outputs should be parseable by the external command feature."
+
diff --git a/src/test/regression/tests/041.external_replication_delay/test_validation.sh b/src/test/regression/tests/041.external_replication_delay/test_validation.sh
new file mode 100644
index 000000000..2ea8e32b8
--- /dev/null
+++ b/src/test/regression/tests/041.external_replication_delay/test_validation.sh
@@ -0,0 +1,286 @@
+#!/usr/bin/env bash
+#-------------------------------------------------------------------
+# test script for external command validation and edge cases
+#
+source $TESTLIBS
+TESTDIR=testdir_validation
+PG_CTL=$PGBIN/pg_ctl
+PSQL="$PGBIN/psql -X "
+
+rm -fr $TESTDIR
+mkdir $TESTDIR
+cd $TESTDIR
+
+# create test environmen
+echo -n "creating test environment..."
+$PGPOOL_SETUP -m s -n 3 || exit 1
+echo "done."
+source ./bashrc.ports
+export PGPORT=$PGPOOL_PORT
+
+# Create test command scripts
+cat > delay_cmd_validation.sh << 'EOF'
+#!/bin/bash
+# Test validation: output with invalid values
+echo "0 invalid_value 50.5"
+EOF
+chmod +x delay_cmd_validation.sh
+
+cat > delay_cmd_negative.sh << 'EOF'
+#!/bin/bash
+# Test negative values
+echo "0 -25 50"
+EOF
+chmod +x delay_cmd_negative.sh
+
+cat > delay_cmd_large.sh << 'EOF'
+#!/bin/bash
+# Test extremely large values
+echo "0 9999999 50"
+EOF
+chmod +x delay_cmd_large.sh
+
+cat > delay_cmd_wrong_count.sh << 'EOF'
+#!/bin/bash
+# Test wrong number of values (only 2 instead of 3)
+echo "0 25"
+EOF
+chmod +x delay_cmd_wrong_count.sh
+
+cat > delay_cmd_truncated.sh << 'EOF'
+#!/bin/bash
+# Test output that might be truncated (very long line)
+printf "0 25 "
+for i in {1..1000}; do printf "very_long_output_"; done
+echo "50"
+EOF
+chmod +x delay_cmd_truncated.sh
+
+# ----------------------------------------------------------------------------------------
+echo === Test1: Validation of invalid delay values ===
+# ----------------------------------------------------------------------------------------
+echo "replication_delay_source = 'cmd'" >> etc/pgpool.conf
+echo "replication_delay_source_cmd = './delay_cmd_validation.sh'" >> etc/pgpool.conf
+echo "sr_check_period = 1" >> etc/pgpool.conf
+echo "log_standby_delay = 'always'" >> etc/pgpool.conf
+echo "log_min_messages = 'DEBUG1'" >> etc/pgpool.conf
+
+./startall
+wait_for_pgpool_startup
+
+$PSQL test <<EOF
+CREATE TABLE t1(i INTEGER);
+EOF
+
+# Wait for sr_check to run
+echo "Waiting for validation test..."
+for i in {1..10}; do
+    if grep -q "invalid delay value" log/pgpool.log 2>/dev/null; then
+        echo "Validation error detected after ${i} seconds"
+        break
+    fi
+    sleep 1
+done
+
+# Check for validation warning
+grep "invalid delay value 'invalid_value' for node" log/pgpool.log >/dev/null 2>&1
+if [ $? != 0 ];then
+    echo fail: validation warning not found
+    ./shutdownall
+    exit 1
+fi
+
+echo ok: invalid value validation test succeeded
+./shutdownall
+
+# ----------------------------------------------------------------------------------------
+echo === Test2: Negative delay values ===
+# ----------------------------------------------------------------------------------------
+sed -i.bak "s|delay_cmd_validation.sh|delay_cmd_negative.sh|" etc/pgpool.conf
+
+./startall
+wait_for_pgpool_startup
+
+# Wait for sr_check to run
+echo "Waiting for negative value test..."
+for i in {1..10}; do
+    if grep -q "negative delay value" log/pgpool.log 2>/dev/null; then
+        echo "Negative value warning detected after ${i} seconds"
+        break
+    fi
+    sleep 1
+done
+
+# Check for negative value warning
+grep "negative delay value.*for node" log/pgpool.log >/dev/null 2>&1
+if [ $? != 0 ];then
+    echo fail: negative value warning not found
+    ./shutdownall
+    exit 1
+fi
+
+echo ok: negative value validation test succeeded
+./shutdownall
+
+# ----------------------------------------------------------------------------------------
+echo === Test3: Extremely large delay values ===
+# ----------------------------------------------------------------------------------------
+sed -i.bak "s|delay_cmd_negative.sh|delay_cmd_large.sh|" etc/pgpool.conf
+
+./startall
+wait_for_pgpool_startup
+
+# Wait for sr_check to run
+echo "Waiting for large value test..."
+for i in {1..10}; do
+    if grep -q "extremely large delay value" log/pgpool.log 2>/dev/null; then
+        echo "Large value warning detected after ${i} seconds"
+        break
+    fi
+    sleep 1
+done
+
+# Check for large value warning
+grep "extremely large delay value.*for node" log/pgpool.log >/dev/null 2>&1
+if [ $? != 0 ];then
+    echo fail: large value warning not found
+    ./shutdownall
+    exit 1
+fi
+
+echo ok: large value validation test succeeded
+./shutdownall
+
+# ----------------------------------------------------------------------------------------
+echo === Test4: Wrong number of output values ===
+# ----------------------------------------------------------------------------------------
+sed -i.bak "s|delay_cmd_large.sh|delay_cmd_wrong_count.sh|" etc/pgpool.conf
+
+./startall
+wait_for_pgpool_startup
+
+# Wait for sr_check to run
+echo "Waiting for wrong count test..."
+for i in {1..10}; do
+    if grep -q "returned.*values, expected" log/pgpool.log 2>/dev/null; then
+        echo "Wrong count warning detected after ${i} seconds"
+        break
+    fi
+    sleep 1
+done
+
+# Check for wrong count warning
+grep "returned.*values, expected.*Command should output one delay value per backend node" log/pgpool.log >/dev/null 2>&1
+if [ $? != 0 ];then
+    echo fail: wrong count validation test not found
+    ./shutdownall
+    exit 1
+fi
+
+echo ok: wrong count validation test succeeded
+./shutdownall
+
+# ----------------------------------------------------------------------------------------
+echo === Test5: Primary node non-zero delay handling ===
+# ----------------------------------------------------------------------------------------
+cat > delay_cmd_primary_nonzero.sh << 'EOF'
+#!/bin/bash
+# Test primary node with non-zero delay (should be corrected to 0)
+echo "100 25 50"
+EOF
+chmod +x delay_cmd_primary_nonzero.sh
+
+sed -i.bak "s|delay_cmd_wrong_count.sh|delay_cmd_primary_nonzero.sh|" etc/pgpool.conf
+
+./startall
+wait_for_pgpool_startup
+
+# Wait for sr_check to run
+echo "Waiting for primary non-zero delay test..."
+for i in {1..10}; do
+    if grep -q "primary node.*reported non-zero delay" log/pgpool.log 2>/dev/null; then
+        echo "Primary non-zero delay detected after ${i} seconds"
+        break
+    fi
+    sleep 1
+done
+
+# Check for primary node correction
+grep "primary node.*reported non-zero delay.*setting to 0" log/pgpool.log >/dev/null 2>&1
+if [ $? != 0 ];then
+    echo fail: primary node delay correction not found
+    ./shutdownall
+    exit 1
+fi
+
+echo ok: primary node delay correction test succeeded
+./shutdownall
+
+# ----------------------------------------------------------------------------------------
+echo === Test6: Command timeout with different timeout values ===
+# ----------------------------------------------------------------------------------------
+cat > delay_cmd_timeout.sh << 'EOF'
+#!/bin/bash
+# Command that takes 5 seconds
+sleep 5
+echo "0 25 50"
+EOF
+chmod +x delay_cmd_timeout.sh
+
+# Test with timeout shorter than command duration
+sed -i.bak "s|delay_cmd_primary_nonzero.sh|delay_cmd_timeout.sh|" etc/pgpool.conf
+echo "replication_delay_source_timeout = 2" >> etc/pgpool.conf
+
+./startall
+wait_for_pgpool_startup
+
+# Wait for timeou
+echo "Waiting for timeout test (2s timeout, 5s command)..."
+for i in {1..10}; do
+    if grep -q "replication delay command timed out after 2 seconds" log/pgpool.log 2>/dev/null; then
+        echo "Timeout detected after ${i} seconds"
+        break
+    fi
+    sleep 1
+done
+
+# Check for timeout message
+grep "replication delay command timed out after 2 seconds" log/pgpool.log >/dev/null 2>&1
+if [ $? != 0 ];then
+    echo fail: timeout not detected
+    ./shutdownall
+    exit 1
+fi
+
+echo ok: timeout test succeeded
+./shutdownall
+
+# Test with timeout longer than command duration
+sed -i.bak "s|replication_delay_source_timeout = 2|replication_delay_source_timeout = 10|" etc/pgpool.conf
+
+./startall
+wait_for_pgpool_startup
+
+# Wait for successful execution
+echo "Waiting for successful execution (10s timeout, 5s command)..."
+for i in {1..15}; do
+    if grep -q "executing replication delay command.*delay_cmd_timeout.sh" log/pgpool.log 2>/dev/null; then
+        echo "Command executed successfully after ${i} seconds"
+        break
+    fi
+    sleep 1
+done
+
+# Should not timeout this time
+if grep -q "replication delay command timed out" log/pgpool.log 2>/dev/null; then
+    echo fail: command should not have timed out with 10s timeou
+    ./shutdownall
+    exit 1
+fi
+
+echo ok: extended timeout test succeeded
+./shutdownall
+
+echo "All validation tests passed!"
+exit 0
+
-- 
2.51.0

