supabase-postgres.git / summary / log / commit / refs

commit    bc12563f245ebe4b7669e08f86f414864fec55ce
Author:   Douglas J Hunley <doug.hunley@gmail.com>
Commit:   GitHub <noreply@github.com>
Date:     Fri Jul 17 15:54:09 2026 +0000

    Check for 2 GB free space before PG upgrade (#2271)
    
    * fix(pg_upgrade): check for 2 GiB free space on / before running upgrade
    
    initiate.sh realizes the new pg_upgrade Nix store path onto / before
    touching the mounted data disk. If / is already tight on space, Nix
    fills the partition mid-upgrade and the failure surfaces deep into the
    run instead of at the start. check_free_space in common.sh checks
    available space on / via df and aborts immediately, before any
    mkdir/nix work, if less than 2 GiB is free.
    
    * fix: fail closed on unparseable df output in check_free_space
    
    df -k wraps onto a second line when the source device name is long, which misaligns the NR==2 awk parse; -P forces single-line POSIX output. Also validate available_kb is numeric before comparing — an empty/non-numeric value from a bad df parse made the bash [ -lt ] test error inside an if condition, which set -e doesn't catch, silently skipping the guard instead of aborting.
    
    * style: match treefmt/shfmt quoting for [[ =~ ]] test
    
    * style(pg_upgrade): use bash arithmetic comparison in check_free_space
    
    [ -lt ] swapped for (( < )) per review — script is bash, not posix sh,
    so arithmetic mode reads clearer.
    
    * fix: strip inner spaces from arithmetic conditional in check_free_space
    
    treefmt's shfmt formatting normalizes `(( expr ))` to `((expr))` with no
    inner padding; the added check_free_space guard used the padded form,
    failing pre-commit-run/treefmt-check CI on all platforms.
    
    * fix(pg_upgrade): re-check free space around nix-store realize
    
    check_free_space only ran once, at the top of initiate_upgrade, but the
    tarball extraction, Nix install, and catalog download that follow can eat
    into that 2 GiB reserve before nix-store -r (the actual big consumer) runs,
    letting a host that passed the entry check still hit ENOSPC mid-realize.
    
    Add a second 2 GiB check immediately before the nix-store -r retry loop to
    confirm the reserve is still intact right where it's needed, and a smaller
    512 MiB check after the closure is realized, since the remaining apt-get/
    locale-gen work needs far less headroom than the Nix realize did.
    
    * fix(pg_upgrade): guard cleanup's umount so status file isn't stuck at running
    
    check_free_space runs before $MOUNT_POINT is ever mounted, so when it fails
    the ERR trap's cleanup() hits an umount on an unmounted path. Every other
    risky command in cleanup() already has || true; this one didn't, so under
    set -e the trap handler aborted before writing UPGRADE_STATUS, leaving
    /tmp/pg-upgrade-status stuck at "running" forever on exactly the low-disk
    failure path this PR adds. Reproduced and confirmed fixed with a minimal
    harness matching the trap structure (removed after verifying).


 .../admin_api_scripts/pg_upgrade_scripts/common.sh  | 21 +++++++++++++++++++++
 .../pg_upgrade_scripts/initiate.sh                  | 20 +++++++++++++++++++-
 2 files changed, 40 insertions(+), 1 deletion(-)

diff --git a/ansible/files/admin_api_scripts/pg_upgrade_scripts/common.sh b/ansible/files/admin_api_scripts/pg_upgrade_scripts/common.sh
index a3f6f208..699d7aeb 100755
--- a/ansible/files/admin_api_scripts/pg_upgrade_scripts/common.sh
+++ b/ansible/files/admin_api_scripts/pg_upgrade_scripts/common.sh
@@ -44,6 +44,27 @@ function ship_logs {
 		-d "$BODY"
 }
 
+# Aborts if / has less than required_kb free. Nix realizes the new pg_upgrade
+# store path onto / before the mounted data disk is touched, so a full root
+# partition otherwise fails the upgrade mid-flight instead of at the start.
+function check_free_space {
+	local required_kb=$1
+	local available_kb
+	# -P forces POSIX single-line output; plain `df -k` wraps onto a second
+	# line when the source device name is long, which would misalign NR==2.
+	available_kb=$(df -Pk / | awk 'NR==2 {print $4}')
+
+	if ! [[ $available_kb =~ ^[0-9]+$ ]]; then
+		echo "ERROR: could not determine free space on /; aborting upgrade."
+		return 1
+	fi
+
+	if ((available_kb < required_kb)); then
+		echo "ERROR: only ${available_kb}KB free on / but ${required_kb}KB required; aborting upgrade."
+		return 1
+	fi
+}
+
 function retry {
 	local retries=$1
 	shift
diff --git a/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh b/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh
index df37550a..1b100c9a 100755
--- a/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh
+++ b/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh
@@ -156,7 +156,11 @@ EOF
 
 	if [ -z "$IS_CI" ] && [ -z "$IS_LOCAL_UPGRADE" ]; then
 		echo "Unmounting data disk from ${MOUNT_POINT}"
-		retry 3 umount $MOUNT_POINT
+		# check_free_space (and other pre-mount failures) can trigger this trap before
+		# $MOUNT_POINT is ever mounted. Without || true, set -e aborts cleanup() right here
+		# under retry's non-zero exit, so UPGRADE_STATUS is never written and the status file
+		# is stuck at "running" forever instead of reporting the real failure.
+		retry 3 umount $MOUNT_POINT || true
 	fi
 	echo "$UPGRADE_STATUS" >/tmp/pg-upgrade-status
 
@@ -211,6 +215,9 @@ EOF
 }
 
 function initiate_upgrade {
+	# 2 GiB: enough headroom for the Nix store realize onto / (see check_free_space)
+	check_free_space $((2 * 1024 * 1024))
+
 	mkdir -p "$MOUNT_POINT"
 	SHARED_PRELOAD_LIBRARIES=$(grep '^[[:space:]]*shared_preload_libraries' "$POSTGRES_CONFIG_PATH" | sed "s/shared_preload_libraries =\s\{0,1\}'\(.*\)'.*/\1/")
 
@@ -356,6 +363,13 @@ EXTRA_NIX_CONF
 		# registered paths are skipped on retry but an in-flight NAR restarts from
 		# 0%. Failing as a normal command (not exit 1) lets the ERR trap run cleanup
 		# and record "failed" instead of hanging.
+		#
+		# Re-check free space here: the earlier check at the top of initiate_upgrade
+		# only guarantees headroom at entry, and the tarball extraction / nix install
+		# / catalog download above can already have eaten into that reserve before
+		# the realize (the actual big consumer) starts.
+		check_free_space $((2 * 1024 * 1024))
+
 		nix_store_ok="false"
 		for attempt in 1 2 3; do
 			if timeout -k 10s 120s nix-store -r "$STORE_PATH"; then
@@ -370,6 +384,10 @@ EXTRA_NIX_CONF
 		done
 		[ "$nix_store_ok" = "true" ]
 
+		# Closure is realized, so the multi-GB Nix reserve is no longer needed; just
+		# confirm enough room remains for the lighter apt/locale-gen work still ahead.
+		check_free_space $((512 * 1024))
+
 		PG_UPGRADE_BIN_DIR="$STORE_PATH"
 		PGSHARENEW="$PG_UPGRADE_BIN_DIR/share/postgresql"
 	fi


[parent: 0c68cac1da5e]