supabase-postgres.git / summary / log / commit / refs
commit 70657e2f3b7ea7ac66031ab324f4db20c937d67f
Author: samrose <samuel@supabase.io>
Commit: GitHub <noreply@github.com>
Date: Tue Mar 18 20:46:08 2025 +0000
feature: wal-g v 3 (#1456)
* feature: wal-g v 3
* chore: install, symlink
* chore: wal-g user and groups in stage 1
* chore: bump versions to test wal-g
* chore: build wal-g in ci and copy to binary cache on success
* chore: minor correction
* chore: maintainer
* chore: release wal-g 3
ansible/playbook.yml | 2 +-
ansible/tasks/setup-wal-g.yml | 142 +++++++++++++++++++++++-------------------
ansible/vars.yml | 5 +-
docker/nix/build_nix.sh | 4 +-
flake.nix | 2 +
nix/wal-g.nix | 58 +++++++++++++++++
6 files changed, 144 insertions(+), 69 deletions(-)
diff --git a/ansible/playbook.yml b/ansible/playbook.yml
index 5c5f8da3..5c8a9f70 100644
--- a/ansible/playbook.yml
+++ b/ansible/playbook.yml
@@ -38,7 +38,7 @@
- name: Install WAL-G
import_tasks: tasks/setup-wal-g.yml
- when: debpkg_mode or nixpkg_mode
+ when: debpkg_mode or nixpkg_mode or stage2_nix
- name: Install Gotrue
import_tasks: tasks/setup-gotrue.yml
diff --git a/ansible/tasks/setup-wal-g.yml b/ansible/tasks/setup-wal-g.yml
index bbc64cdd..9907d3ee 100644
--- a/ansible/tasks/setup-wal-g.yml
+++ b/ansible/tasks/setup-wal-g.yml
@@ -1,72 +1,73 @@
# Downloading dependencies
-- name: wal-g dependencies
- become: yes
- apt:
- pkg:
- - libbrotli-dev
- - liblzo2-dev
- - libsodium-dev
- - cmake
+# - name: wal-g dependencies
+# become: yes
+# apt:
+# pkg:
+# - libbrotli-dev
+# - liblzo2-dev
+# - libsodium-dev
+# - cmake
# install go dependency for WAL-G
-- name: wal-g go dependency
- get_url:
- url: "https://golang.org/dl/go{{ golang_version }}.linux-{{ platform }}.tar.gz"
- dest: /tmp
- checksum: "{{ golang_version_checksum[platform] }}"
- timeout: 60
+# - name: wal-g go dependency
+# get_url:
+# url: "https://golang.org/dl/go{{ golang_version }}.linux-{{ platform }}.tar.gz"
+# dest: /tmp
+# checksum: "{{ golang_version_checksum[platform] }}"
+# timeout: 60
-- name: unpack go archive
- unarchive:
- remote_src: yes
- src: "/tmp/go{{ golang_version }}.linux-{{ platform }}.tar.gz"
- dest: /usr/local
+# - name: unpack go archive
+# unarchive:
+# remote_src: yes
+# src: "/tmp/go{{ golang_version }}.linux-{{ platform }}.tar.gz"
+# dest: /usr/local
-# Download WAL-G
-- name: wal-g - download latest version
- git:
- repo: https://github.com/wal-g/wal-g.git
- dest: /tmp/wal-g
- version: "v{{ wal_g_release }}"
- become: yes
+# # Download WAL-G
+# - name: wal-g - download latest version
+# git:
+# repo: https://github.com/wal-g/wal-g.git
+# dest: /tmp/wal-g
+# version: "v{{ wal_g_release }}"
+# become: yes
-- name: wal-g - pg_clean
- make:
- chdir: /tmp/wal-g
- target: pg_clean
- params:
- GOBIN: "/usr/local/bin"
- PATH: "{{ ansible_env.PATH }}:/usr/local/go/bin"
- USE_LIBSODIUM: true
- become: yes
- ignore_errors: yes
+# - name: wal-g - pg_clean
+# make:
+# chdir: /tmp/wal-g
+# target: pg_clean
+# params:
+# GOBIN: "/usr/local/bin"
+# PATH: "{{ ansible_env.PATH }}:/usr/local/go/bin"
+# USE_LIBSODIUM: true
+# become: yes
+# ignore_errors: yes
-- name: wal-g - deps
- make:
- chdir: /tmp/wal-g
- target: deps
- params:
- GOBIN: "/usr/local/bin"
- PATH: "{{ ansible_env.PATH }}:/usr/local/go/bin"
- USE_LIBSODIUM: true
- become: yes
- ignore_errors: yes
+# - name: wal-g - deps
+# make:
+# chdir: /tmp/wal-g
+# target: deps
+# params:
+# GOBIN: "/usr/local/bin"
+# PATH: "{{ ansible_env.PATH }}:/usr/local/go/bin"
+# USE_LIBSODIUM: true
+# become: yes
+# ignore_errors: yes
-- name: wal-g - build and install
- community.general.make:
- chdir: /tmp/wal-g
- target: pg_install
- jobs: "{{ parallel_jobs | default(omit) }}"
- params:
- GOBIN: "/usr/local/bin"
- PATH: "{{ ansible_env.PATH }}:/usr/local/go/bin"
- USE_LIBSODIUM: true
- become: yes
+# - name: wal-g - build and install
+# community.general.make:
+# chdir: /tmp/wal-g
+# target: pg_install
+# jobs: "{{ parallel_jobs | default(omit) }}"
+# params:
+# GOBIN: "/usr/local/bin"
+# PATH: "{{ ansible_env.PATH }}:/usr/local/go/bin"
+# USE_LIBSODIUM: true
+# become: yes
- name: Create wal-g group
group:
name: wal-g
state: present
+ when: nixpkg_mode
- name: Create wal-g user
user:
@@ -75,7 +76,7 @@
comment: WAL-G user
group: wal-g
groups: wal-g, postgres
-
+ when: nixpkg_mode
- name: Create a config directory owned by wal-g
file:
path: /etc/wal-g
@@ -83,7 +84,22 @@
owner: wal-g
group: wal-g
mode: '0770'
+ when: nixpkg_mode
+- name: Install wal-g from nix binary cache
+ become: yes
+ shell: |
+ sudo -u wal-g bash -c ". /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && nix profile install github:supabase/postgres/{{ git_commit_sha }}#wal-g"
+ when: stage2_nix
+
+- name: Create symlink for wal-g from Nix profile to /usr/local/bin
+ ansible.builtin.file:
+ src: /home/wal-g/.nix-profile/bin/wal-g
+ dest: /usr/local/bin/wal-g
+ state: link
+ force: yes # This will replace existing file/symlink if it exists
+ become: yes # Need sudo to write to /usr/local/bin
+ when: stage2_nix
- name: Create /etc/wal-g/config.json
file:
path: /etc/wal-g/config.json
@@ -91,6 +107,7 @@
owner: wal-g
group: wal-g
mode: '0664'
+ when: stage2_nix
- name: Move custom wal-g.conf file to /etc/postgresql-custom/wal-g.conf
template:
@@ -99,6 +116,7 @@
mode: 0664
owner: postgres
group: postgres
+ when: stage2_nix
- name: Add script to be run for restore_command
template:
@@ -107,6 +125,7 @@
mode: 0500
owner: postgres
group: postgres
+ when: stage2_nix
- name: Add helper script for wal_fetch.sh
template:
@@ -114,6 +133,7 @@
dest: /root/wal_change_ownership.sh
mode: 0700
owner: root
+ when: stage2_nix
- name: Include /etc/postgresql-custom/wal-g.conf in postgresql.conf
become: yes
@@ -121,10 +141,4 @@
path: /etc/postgresql/postgresql.conf
regexp: "#include = '/etc/postgresql-custom/wal-g.conf'"
replace: "include = '/etc/postgresql-custom/wal-g.conf'"
-
-# Clean up Go
-- name: Uninstall Go
- become: yes
- file:
- path: /usr/local/go
- state: absent
+ when: stage2_nix
diff --git a/ansible/vars.yml b/ansible/vars.yml
index 381fb951..d8e963b7 100644
--- a/ansible/vars.yml
+++ b/ansible/vars.yml
@@ -8,8 +8,8 @@ postgres_major:
# Full version strings for each major version
postgres_release:
- postgresorioledb-17: "17.0.1.048-orioledb"
- postgres15: "15.8.1.055"
+ postgresorioledb-17: "17.0.1.049-orioledb"
+ postgres15: "15.8.1.056"
# Non Postgres Extensions
pgbouncer_release: "1.19.0"
@@ -45,7 +45,6 @@ kong_deb_checksum: sha1:2086f6ccf8454fe64435252fea4d29d736d7ec61
nginx_release: 1.22.0
nginx_release_checksum: sha1:419efb77b80f165666e2ee406ad8ae9b845aba93
-wal_g_release: "2.0.1"
postgres_exporter_release: "0.15.0"
postgres_exporter_release_checksum:
diff --git a/docker/nix/build_nix.sh b/docker/nix/build_nix.sh
index b0eef986..48286907 100644
--- a/docker/nix/build_nix.sh
+++ b/docker/nix/build_nix.sh
@@ -13,8 +13,9 @@ nix build .#checks.$SYSTEM.psql_orioledb-17 -L --no-link
nix build .#psql_15/bin -o psql_15
nix build .#psql_orioledb-17/bin -o psql_orioledb_17
-
+nix build .#wal-g -o wal-g
# Copy to S3
+nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key ./wal-g
nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key ./psql_15
nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key ./psql_orioledb_17
if [ "$SYSTEM" = "aarch64-linux" ]; then
@@ -26,4 +27,5 @@ if [ "$SYSTEM" = "aarch64-linux" ]; then
nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key ./postgresql_15_src
nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key ./postgresql_orioledb-17_debug-debug
nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key ./postgresql_orioledb-17_src
+ nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key ./wal-g
fi
diff --git a/flake.nix b/flake.nix
index 3abdbaff..ba7ad70e 100644
--- a/flake.nix
+++ b/flake.nix
@@ -87,6 +87,7 @@
sfcgal = pkgs.callPackage ./nix/ext/sfcgal/sfcgal.nix { };
supabase-groonga = pkgs.callPackage ./nix/supabase-groonga.nix { };
mecab-naist-jdic = pkgs.callPackage ./nix/ext/mecab-naist-jdic/default.nix { };
+ wal-g = pkgs.callPackage ./nix/wal-g.nix { };
# Our list of PostgreSQL extensions which come from upstream Nixpkgs.
# These are maintained upstream and can easily be used here just by
# listing their name. Anytime the version of nixpkgs is upgraded, these
@@ -405,6 +406,7 @@
# PostgreSQL versions.
psql_15 = postgresVersions.psql_15;
psql_orioledb-17 = postgresVersions.psql_orioledb-17;
+ wal-g = wal-g;
sfcgal = sfcgal;
pg_prove = pkgs.perlPackages.TAPParserSourceHandlerpgTAP;
inherit postgresql_15 postgresql_orioledb-17;
diff --git a/nix/wal-g.nix b/nix/wal-g.nix
new file mode 100644
index 00000000..13f2bbc9
--- /dev/null
+++ b/nix/wal-g.nix
@@ -0,0 +1,58 @@
+{
+ lib,
+ buildGoModule,
+ fetchFromGitHub,
+ brotli,
+ libsodium,
+ installShellFiles,
+}:
+
+buildGoModule rec {
+ pname = "wal-g";
+ version = "3.0.5";
+
+ src = fetchFromGitHub {
+ owner = "wal-g";
+ repo = "wal-g";
+ rev = "v${version}";
+ sha256 = "sha256-wVr0L2ZXMuEo6tc2ajNzPinVQ8ZVzNOSoaHZ4oFsA+U=";
+ };
+
+ vendorHash = "sha256-YDLAmRfDl9TgbabXj/1rxVQ052NZDg3IagXVTe5i9dw=";
+
+ nativeBuildInputs = [ installShellFiles ];
+
+ buildInputs = [
+ brotli
+ libsodium
+ ];
+
+ subPackages = [ "main/pg" ];
+
+ tags = [
+ "brotli"
+ "libsodium"
+ ];
+
+ ldflags = [
+ "-s"
+ "-w"
+ "-X github.com/wal-g/wal-g/cmd/pg.walgVersion=${version}"
+ "-X github.com/wal-g/wal-g/cmd/pg.gitRevision=${src.rev}"
+ ];
+
+ postInstall = ''
+ mv $out/bin/pg $out/bin/wal-g
+ installShellCompletion --cmd wal-g \
+ --bash <($out/bin/wal-g completion bash) \
+ --zsh <($out/bin/wal-g completion zsh)
+ '';
+
+ meta = with lib; {
+ homepage = "https://github.com/wal-g/wal-g";
+ license = licenses.asl20;
+ description = "Archival restoration tool for PostgreSQL";
+ mainProgram = "wal-g";
+ maintainers = [ samrose ];
+ };
+}
[parent: bf7327c13554]