supabase-postgres.git / summary / log / commit / refs
commit e2b91a25cfcb952a807d35ec1955067efd9004ef
Author: Manuel Mendez <708570+mmlb@users.noreply.github.com>
Commit: GitHub <noreply@github.com>
Date: Fri Jul 10 21:46:42 2026 +0000
Fix flaky initdb step in stage2 of AMI build (#2263)
* ansible: Remove left over debug tag in install-salt
* ansible: Drop "Check if psql_version is psql_orioledb"
* ami: Move /data nofail to stage2
Having this done in stage1 becomes a problem when booting the stage2
instance. The original intent of this was for it to apply for runtime
boots, not build time.
This is causing CI to be flaky and fail with:
> initdb: error: could not create directory "/var/lib/postgresql/data": File exists
> pg_ctl: database system initialization failed
Replicated the isssue using postgres:15 docker image:
```
$ # need --privileged to deal with /var/lib/postgresql/data VOLUME
$ docker run --rm -ti --entrypoint=bash --privileged --hostname=pgtest postgres:15
root@pgtest:/# su postgres
postgres@pgtest:/$ /usr/lib/postgresql/15/bin/pg_ctl -D /var/lib/postgresql/data initdb -o --allow-group-access -o --username=supabase_admin -o --data-checksums
The files belonging to this database system will be owned by user "postgres".
This user must also own the server process.
The database cluster will be initialized with locale "en_US.utf8".
The default database encoding has accordingly been set to "UTF8".
The default text search configuration will be set to "english".
Data page checksums are enabled.
fixing permissions on existing directory /var/lib/postgresql/data ... ok
creating subdirectories ... ok
selecting dynamic shared memory implementation ... posix
selecting default max_connections ... 100
selecting default shared_buffers ... 128MB
selecting default time zone ... Etc/UTC
creating configuration files ... ok
running bootstrap script ... ok
performing post-bootstrap initialization ... ok
syncing data to disk ... ok
initdb: warning: enabling "trust" authentication for local connections
initdb: hint: You can change this by editing pg_hba.conf or using the option -A, or --auth-local and --auth-host, the next time you run initdb.
Success. You can now start the database server using:
/usr/lib/postgresql/15/bin/pg_ctl -D /var/lib/postgresql/data -l logfile start
postgres@pgtest:/$ exit
exit
root@pgtest:/#
root@pgtest:/#
root@pgtest:/#
root@pgtest:/#
root@pgtest:/# # now unmount and replicate our setup with happy path
root@pgtest:/#
root@pgtest:/#
root@pgtest:/#
root@pgtest:/#
root@pgtest:/# umount /var/lib/postgresql/data/
root@pgtest:/# rm -rf /var/lib/postgresql/data/
root@pgtest:/# mkdir /data/pgdata -p
root@pgtest:/# chown -R postgres:postgres /data/pgdata/
root@pgtest:/# su postgres
postgres@pgtest:~$ ln -nsf /data/pgdata/ /var/lib/postgresql/data
postgres@pgtest:~$ /usr/lib/postgresql/15/bin/pg_ctl -D /var/lib/postgresql/data initdb -o --allow-group-access -o --username=supabase_admin -o --data-checksums
The files belonging to this database system will be owned by user "postgres".
This user must also own the server process.
The database cluster will be initialized with locale "en_US.utf8".
The default database encoding has accordingly been set to "UTF8".
The default text search configuration will be set to "english".
Data page checksums are enabled.
fixing permissions on existing directory /var/lib/postgresql/data ... ok
creating subdirectories ... ok
selecting dynamic shared memory implementation ... posix
selecting default max_connections ... 100
selecting default shared_buffers ... 128MB
selecting default time zone ... Etc/UTC
creating configuration files ... ok
running bootstrap script ... ok
performing post-bootstrap initialization ... ok
syncing data to disk ... ok
initdb: warning: enabling "trust" authentication for local connections
initdb: hint: You can change this by editing pg_hba.conf or using the option -A, or --auth-local and --auth-host, the next time you run initdb.
Success. You can now start the database server using:
/usr/lib/postgresql/15/bin/pg_ctl -D /var/lib/postgresql/data -l logfile start
postgres@pgtest:~$ exit
exit
root@pgtest:/#
root@pgtest:/#
root@pgtest:/#
root@pgtest:/#
root@pgtest:/# # now delete /data/pgdata to simulate skipped /data mount
root@pgtest:/#
root@pgtest:/#
root@pgtest:/#
root@pgtest:/#
root@pgtest:/# rm -rf /data/pgdata/
root@pgtest:/# su postgres
postgres@pgtest:/$ /usr/lib/postgresql/15/bin/pg_ctl -D /var/lib/postgresql/data initdb -o --allow-group-access -o --username=supabase_admin -o --data-checksums
The files belonging to this database system will be owned by user "postgres".
This user must also own the server process.
The database cluster will be initialized with locale "en_US.utf8".
The default database encoding has accordingly been set to "UTF8".
The default text search configuration will be set to "english".
Data page checksums are enabled.
creating directory /var/lib/postgresql/data ... initdb: error: could not create directory "/var/lib/postgresql/data": File exists
pg_ctl: database system initialization failed
postgres@pgtest:/$ exit
exit
root@pgtest:/#
exit
```
ansible/tasks/internal/install-salt.yml | 1 -
ansible/tasks/setup-postgres.yml | 28 +++++++++++++++----------
ebssurrogate/scripts/surrogate-bootstrap-nix.sh | 2 +-
3 files changed, 18 insertions(+), 13 deletions(-)
diff --git a/ansible/tasks/internal/install-salt.yml b/ansible/tasks/internal/install-salt.yml
index e80d219f..47436ded 100644
--- a/ansible/tasks/internal/install-salt.yml
+++ b/ansible/tasks/internal/install-salt.yml
@@ -57,7 +57,6 @@
update_cache: yes
- name: Pin salt packages at major version
- tags: mmlb
ansible.builtin.copy:
content: |
Package: salt-common salt-minion
diff --git a/ansible/tasks/setup-postgres.yml b/ansible/tasks/setup-postgres.yml
index be63650d..e38026b4 100644
--- a/ansible/tasks/setup-postgres.yml
+++ b/ansible/tasks/setup-postgres.yml
@@ -250,16 +250,23 @@
when:
- nixpkg_mode
-- name: Check psql_version and modify supautils.conf and postgresql.conf if necessary
+- name: Ensure /data configured for production runtime
+ when:
+ - stage2_nix
+ ansible.posix.mount:
+ src: "UUID={{ ansible_mounts | selectattr('mount', 'equalto', '/data') | map(attribute='uuid') | first }}"
+ path: /data
+ fstype: ext4
+ opts: defaults,discard,nofail,x-systemd.device-timeout=5s
+ dump: "0"
+ passno: "2"
+ state: mounted
+
+- name: Run initdb (stage2_nix)
when:
- stage2_nix
block:
- - name: Check if psql_version is psql_orioledb
- ansible.builtin.set_fact:
- is_psql_17: "{{ psql_version in ['psql_17'] }}"
- is_psql_oriole: "{{ psql_version in ['psql_orioledb-17'] }}"
-
- - name: Initialize the database stage2_nix (non-orioledb)
+ - name: Initialize the database stage2_nix (PG <17)
become: true
become_user: 'postgres'
ansible.builtin.command:
@@ -273,10 +280,9 @@
vars:
ansible_command_timeout: 60
when:
- - not is_psql_oriole
- - not is_psql_17
+ - psql_version not in ['psql_17', 'psql_orioledb-17']
- - name: Initialize the database stage2_nix (orioledb)
+ - name: Initialize the database stage2_nix (PG >=17)
become: true
become_user: 'postgres'
ansible.builtin.command:
@@ -290,7 +296,7 @@
vars:
ansible_command_timeout: 60
when:
- - (is_psql_oriole or is_psql_17)
+ - psql_version in ['psql_17', 'psql_orioledb-17']
- name: copy PG apparmor profile
ansible.builtin.copy:
diff --git a/ebssurrogate/scripts/surrogate-bootstrap-nix.sh b/ebssurrogate/scripts/surrogate-bootstrap-nix.sh
index 47f3014d..82ff8d80 100755
--- a/ebssurrogate/scripts/surrogate-bootstrap-nix.sh
+++ b/ebssurrogate/scripts/surrogate-bootstrap-nix.sh
@@ -236,7 +236,7 @@ function pull_docker {
function create_fstab {
FMT="%-42s %-11s %-5s %-17s %-5s %s"
local ROOT_LINE=$(findmnt -no SOURCE /mnt | xargs blkid -o export | awk -v FMT="${FMT}" '/^UUID=/ { printf(FMT, $0, "/", "ext4", "defaults,discard", "0", "1" ) }')
- local DATA_LINE=$(findmnt -no SOURCE /mnt/data | xargs blkid -o export | awk -v FMT="${FMT}" '/^UUID=/ { printf(FMT, $0, "/data", "ext4", "defaults,discard,nofail,x-systemd.device-timeout=5s", "0", "2" ) }')
+ local DATA_LINE=$(findmnt -no SOURCE /mnt/data | xargs blkid -o export | awk -v FMT="${FMT}" '/^UUID=/ { printf(FMT, $0, "/data", "ext4", "defaults,discard", "0", "2" ) }')
local SWAP_LINE=$(printf "$FMT" "/swapfile" "none" "swap" "sw" "0" "0")
local EFI_LINE=""