supabase-postgres.git / summary / log / commit / refs
commit 60f8fda593c9c150278b56ad91c65fad19abc95f
Author: samrose <samuel@supabase.io>
Commit: GitHub <noreply@github.com>
Date: Fri May 02 18:19:40 2025 +0000
Sam/build ami local (#1547)
* feature: run testinfra locally
* chore: create way to run testinfra locally
* feature: local ami dev tools
* feat: retry
* feat: run components of testinfra locally as distinct apps
* test: local run of testinfra
* test: specify ami and branch
* feat: pin nixpkgs dep until ready to update
* feat: use ref fo rbranch
* test: push trigger
* test: add premission block
* test: aws creds
* test: run on our arm runner
* feat: no need to check aws-vault on run
* test: running testinfra locally
* chore: roll back inputs for now. change in a future pr
* fix: dynamic port handling since it can collide with parallel test
* feat: make sure ec2 instance terminates at end of run no matter how ended
* docs: document usage
* chore: remove as not needed after all
* chore: newline
testinfra-ami-build.yml} | 0
.gitignore | 1 +
flake.nix | 1569 +++++++++++++-------
nix/docs/development-workflow.md | 141 ++
nix/ext/pg_jsonschema.nix | 5 +-
testinfra/test_ami_nix.py | 203 +--
6 files changed, 1298 insertions(+), 621 deletions(-)
diff --git a/.github/workflows/testinfra-nix.yml b/.github/workflows/testinfra-ami-build.yml
similarity index 100%
rename from .github/workflows/testinfra-nix.yml
rename to .github/workflows/testinfra-ami-build.yml
diff --git a/.gitignore b/.gitignore
index 005d3ece..d32cc8f6 100644
--- a/.gitignore
+++ b/.gitignore
@@ -24,3 +24,4 @@ result*
.vscode/
db/schema.sql
+common-nix.vars.pkr.hcl
diff --git a/flake.nix b/flake.nix
index 70810e19..d49c9e3f 100644
--- a/flake.nix
+++ b/flake.nix
@@ -9,7 +9,7 @@
rust-overlay.url = "github:oxalica/rust-overlay";
};
- outputs = { self, nixpkgs, flake-utils, nix2container, nix-editor, rust-overlay, ...}:
+ outputs = { self, nixpkgs, flake-utils, nix-editor, rust-overlay, nix2container, ... }:
let
gitRev = "vcs=${self.shortRev or "dirty"}+${builtins.substring 0 8 (self.lastModifiedDate or self.lastModified or "19700101")}";
@@ -23,14 +23,13 @@
let
pgsqlDefaultPort = "5435";
pgsqlSuperuser = "supabase_admin";
- nix2img = nix2container.packages.${system}.nix2container;
pkgs = import nixpkgs {
- config = {
+ config = {
allowUnfree = true;
permittedInsecurePackages = [
"v8-9.7.106.18"
- ];
+ ];
};
inherit system;
overlays = [
@@ -84,6 +83,16 @@
})
];
};
+ # Define pythonEnv here
+ pythonEnv = pkgs.python3.withPackages (ps: with ps; [
+ boto3
+ docker
+ pytest
+ pytest-testinfra
+ requests
+ ec2instanceconnectcli
+ paramiko
+ ]);
sfcgal = pkgs.callPackage ./nix/ext/sfcgal/sfcgal.nix { };
supabase-groonga = pkgs.callPackage ./nix/supabase-groonga.nix { };
mecab-naist-jdic = pkgs.callPackage ./nix/ext/mecab-naist-jdic/default.nix { };
@@ -150,15 +159,16 @@
#Where we import and build the orioledb extension, we add on our custom extensions
# plus the orioledb option
#we're not using timescaledb or plv8 in the orioledb-17 version or pg 17 of supabase extensions
- orioleFilteredExtensions = builtins.filter (
- x:
+ orioleFilteredExtensions = builtins.filter
+ (
+ x:
x != ./nix/ext/timescaledb.nix &&
x != ./nix/ext/timescaledb-2.9.1.nix &&
x != ./nix/ext/plv8.nix
) ourExtensions;
orioledbExtensions = orioleFilteredExtensions ++ [ ./nix/ext/orioledb.nix ];
- dbExtensions17 = orioleFilteredExtensions;
+ dbExtensions17 = orioleFilteredExtensions;
getPostgresqlPackage = version:
pkgs.postgresql."postgresql_${version}";
# Create a 'receipt' file for a given postgresql package. This is a way
@@ -196,14 +206,16 @@
};
makeOurPostgresPkgs = version:
- let
+ let
postgresql = getPostgresqlPackage version;
- extensionsToUse = if (builtins.elem version ["orioledb-17"])
+ extensionsToUse =
+ if (builtins.elem version [ "orioledb-17" ])
then orioledbExtensions
- else if (builtins.elem version ["17"])
- then dbExtensions17
+ else if (builtins.elem version [ "17" ])
+ then dbExtensions17
else ourExtensions;
- in map (path: pkgs.callPackage path { inherit postgresql; }) extensionsToUse;
+ in
+ map (path: pkgs.callPackage path { inherit postgresql; }) extensionsToUse;
# Create an attrset that contains all the extensions included in a server.
makeOurPostgresPkgsSet = version:
@@ -252,372 +264,841 @@
recurseForDerivations = true;
};
- makePostgresDevSetup = { pkgs, name, extraSubstitutions ? {} }:
- let
- paths = {
- migrationsDir = builtins.path {
- name = "migrations";
- path = ./migrations/db;
- };
- postgresqlSchemaSql = builtins.path {
- name = "postgresql-schema";
- path = ./nix/tools/postgresql_schema.sql;
- };
- pgbouncerAuthSchemaSql = builtins.path {
- name = "pgbouncer-auth-schema";
- path = ./ansible/files/pgbouncer_config/pgbouncer_auth_schema.sql;
- };
- statExtensionSql = builtins.path {
- name = "stat-extension";
- path = ./ansible/files/stat_extension.sql;
- };
- pgconfigFile = builtins.path {
- name = "postgresql.conf";
- path = ./ansible/files/postgresql_config/postgresql.conf.j2;
- };
- supautilsConfigFile = builtins.path {
- name = "supautils.conf";
- path = ./ansible/files/postgresql_config/supautils.conf.j2;
- };
- loggingConfigFile = builtins.path {
- name = "logging.conf";
- path = ./ansible/files/postgresql_config/postgresql-csvlog.conf;
- };
- readReplicaConfigFile = builtins.path {
- name = "readreplica.conf";
- path = ./ansible/files/postgresql_config/custom_read_replica.conf.j2;
- };
- pgHbaConfigFile = builtins.path {
- name = "pg_hba.conf";
- path = ./ansible/files/postgresql_config/pg_hba.conf.j2;
- };
- pgIdentConfigFile = builtins.path {
- name = "pg_ident.conf";
- path = ./ansible/files/postgresql_config/pg_ident.conf.j2;
- };
- postgresqlExtensionCustomScriptsPath = builtins.path {
- name = "extension-custom-scripts";
- path = ./ansible/files/postgresql_extension_custom_scripts;
- };
- getkeyScript = builtins.path {
- name = "pgsodium_getkey.sh";
- path = ./nix/tests/util/pgsodium_getkey.sh;
+ makePostgresDevSetup = { pkgs, name, extraSubstitutions ? { } }:
+ let
+ paths = {
+ migrationsDir = builtins.path {
+ name = "migrations";
+ path = ./migrations/db;
+ };
+ postgresqlSchemaSql = builtins.path {
+ name = "postgresql-schema";
+ path = ./nix/tools/postgresql_schema.sql;
+ };
+ pgbouncerAuthSchemaSql = builtins.path {
+ name = "pgbouncer-auth-schema";
+ path = ./ansible/files/pgbouncer_config/pgbouncer_auth_schema.sql;
+ };
+ statExtensionSql = builtins.path {
+ name = "stat-extension";
+ path = ./ansible/files/stat_extension.sql;
+ };
+ pgconfigFile = builtins.path {
+ name = "postgresql.conf";
+ path = ./ansible/files/postgresql_config/postgresql.conf.j2;
+ };
+ supautilsConfigFile = builtins.path {
+ name = "supautils.conf";
+ path = ./ansible/files/postgresql_config/supautils.conf.j2;
+ };
+ loggingConfigFile = builtins.path {
+ name = "logging.conf";
+ path = ./ansible/files/postgresql_config/postgresql-csvlog.conf;
+ };
+ readReplicaConfigFile = builtins.path {
+ name = "readreplica.conf";
+ path = ./ansible/files/postgresql_config/custom_read_replica.conf.j2;
+ };
+ pgHbaConfigFile = builtins.path {
+ name = "pg_hba.conf";
+ path = ./ansible/files/postgresql_config/pg_hba.conf.j2;
+ };
+ pgIdentConfigFile = builtins.path {
+ name = "pg_ident.conf";
+ path = ./ansible/files/postgresql_config/pg_ident.conf.j2;
+ };
+ postgresqlExtensionCustomScriptsPath = builtins.path {
+ name = "extension-custom-scripts";
+ path = ./ansible/files/postgresql_extension_custom_scripts;
+ };
+ getkeyScript = builtins.path {
+ name = "pgsodium_getkey.sh";
+ path = ./nix/tests/util/pgsodium_getkey.sh;
+ };
};
- };
-
- localeArchive = if pkgs.stdenv.isDarwin
- then "${pkgs.darwin.locale}/share/locale"
- else "${pkgs.glibcLocales}/lib/locale/locale-archive";
-
- substitutions = {
- SHELL_PATH = "${pkgs.bash}/bin/bash";
- PGSQL_DEFAULT_PORT = "${pgsqlDefaultPort}";
- PGSQL_SUPERUSER = "${pgsqlSuperuser}";
- PSQL15_BINDIR = "${basePackages.psql_15.bin}";
- PSQL17_BINDIR = "${basePackages.psql_17.bin}";
- PSQL_CONF_FILE = "${paths.pgconfigFile}";
- PSQLORIOLEDB17_BINDIR = "${basePackages.psql_orioledb-17.bin}";
- PGSODIUM_GETKEY = "${paths.getkeyScript}";
- READREPL_CONF_FILE = "${paths.readReplicaConfigFile}";
- LOGGING_CONF_FILE = "${paths.loggingConfigFile}";
- SUPAUTILS_CONF_FILE = "${paths.supautilsConfigFile}";
- PG_HBA = "${paths.pgHbaConfigFile}";
- PG_IDENT = "${paths.pgIdentConfigFile}";
- LOCALES = "${localeArchive}";
- EXTENSION_CUSTOM_SCRIPTS_DIR = "${paths.postgresqlExtensionCustomScriptsPath}";
- MECAB_LIB = "${basePackages.psql_15.exts.pgroonga}/lib/groonga/plugins/tokenizers/tokenizer_mecab.so";
- GROONGA_DIR = "${supabase-groonga}";
- MIGRATIONS_DIR = "${paths.migrationsDir}";
- POSTGRESQL_SCHEMA_SQL = "${paths.postgresqlSchemaSql}";
- PGBOUNCER_AUTH_SCHEMA_SQL = "${paths.pgbouncerAuthSchemaSql}";
- STAT_EXTENSION_SQL = "${paths.statExtensionSql}";
- CURRENT_SYSTEM = "${system}";
- } // extraSubstitutions; # Merge in any extra substitutions
- in pkgs.runCommand name {
- inherit (paths) migrationsDir postgresqlSchemaSql pgbouncerAuthSchemaSql statExtensionSql;
- } ''
- set -x
- mkdir -p $out/bin $out/etc/postgresql-custom $out/etc/postgresql $out/extension-custom-scripts
+
+ localeArchive =
+ if pkgs.stdenv.isDarwin
+ then "${pkgs.darwin.locale}/share/locale"
+ else "${pkgs.glibcLocales}/lib/locale/locale-archive";
+
+ substitutions = {
+ SHELL_PATH = "${pkgs.bash}/bin/bash";
+ PGSQL_DEFAULT_PORT = "${pgsqlDefaultPort}";
+ PGSQL_SUPERUSER = "${pgsqlSuperuser}";
+ PSQL15_BINDIR = "${basePackages.psql_15.bin}";
+ PSQL17_BINDIR = "${basePackages.psql_17.bin}";
+ PSQL_CONF_FILE = "${paths.pgconfigFile}";
+ PSQLORIOLEDB17_BINDIR = "${basePackages.psql_orioledb-17.bin}";
+ PGSODIUM_GETKEY = "${paths.getkeyScript}";
+ READREPL_CONF_FILE = "${paths.readReplicaConfigFile}";
+ LOGGING_CONF_FILE = "${paths.loggingConfigFile}";
+ SUPAUTILS_CONF_FILE = "${paths.supautilsConfigFile}";
+ PG_HBA = "${paths.pgHbaConfigFile}";
+ PG_IDENT = "${paths.pgIdentConfigFile}";
+ LOCALES = "${localeArchive}";
+ EXTENSION_CUSTOM_SCRIPTS_DIR = "${paths.postgresqlExtensionCustomScriptsPath}";
+ MECAB_LIB = "${basePackages.psql_15.exts.pgroonga}/lib/groonga/plugins/tokenizers/tokenizer_mecab.so";
+ GROONGA_DIR = "${supabase-groonga}";
+ MIGRATIONS_DIR = "${paths.migrationsDir}";
+ POSTGRESQL_SCHEMA_SQL = "${paths.postgresqlSchemaSql}";
+ PGBOUNCER_AUTH_SCHEMA_SQL = "${paths.pgbouncerAuthSchemaSql}";
+ STAT_EXTENSION_SQL = "${paths.statExtensionSql}";
+ CURRENT_SYSTEM = "${system}";
+ } // extraSubstitutions; # Merge in any extra substitutions
+ in
+ pkgs.runCommand name
+ {
+ inherit (paths) migrationsDir postgresqlSchemaSql pgbouncerAuthSchemaSql statExtensionSql;
+ } ''
+ set -x
+ mkdir -p $out/bin $out/etc/postgresql-custom $out/etc/postgresql $out/extension-custom-scripts
- # Copy config files with error handling
- cp ${paths.supautilsConfigFile} $out/etc/postgresql-custom/supautils.conf || { echo "Failed to copy supautils.conf"; exit 1; }
- cp ${paths.pgconfigFile} $out/etc/postgresql/postgresql.conf || { echo "Failed to copy postgresql.conf"; exit 1; }
- cp ${paths.loggingConfigFile} $out/etc/postgresql-custom/logging.conf || { echo "Failed to copy logging.conf"; exit 1; }
- cp ${paths.readReplicaConfigFile} $out/etc/postgresql-custom/read-replica.conf || { echo "Failed to copy read-replica.conf"; exit 1; }
- cp ${paths.pgHbaConfigFile} $out/etc/postgresql/pg_hba.conf || { echo "Failed to copy pg_hba.conf"; exit 1; }
- cp ${paths.pgIdentConfigFile} $out/etc/postgresql/pg_ident.conf || { echo "Failed to copy pg_ident.conf"; exit 1; }
- cp -r ${paths.postgresqlExtensionCustomScriptsPath}/* $out/extension-custom-scripts/ || { echo "Failed to copy custom scripts"; exit 1; }
+ # Copy config files with error handling
+ cp ${paths.supautilsConfigFile} $out/etc/postgresql-custom/supautils.conf || { echo "Failed to copy supautils.conf"; exit 1; }
+ cp ${paths.pgconfigFile} $out/etc/postgresql/postgresql.conf || { echo "Failed to copy postgresql.conf"; exit 1; }
+ cp ${paths.loggingConfigFile} $out/etc/postgresql-custom/logging.conf || { echo "Failed to copy logging.conf"; exit 1; }
+ cp ${paths.readReplicaConfigFile} $out/etc/postgresql-custom/read-replica.conf || { echo "Failed to copy read-replica.conf"; exit 1; }
+ cp ${paths.pgHbaConfigFile} $out/etc/postgresql/pg_hba.conf || { echo "Failed to copy pg_hba.conf"; exit 1; }
+ cp ${paths.pgIdentConfigFile} $out/etc/postgresql/pg_ident.conf || { echo "Failed to copy pg_ident.conf"; exit 1; }
+ cp -r ${paths.postgresqlExtensionCustomScriptsPath}/* $out/extension-custom-scripts/ || { echo "Failed to copy custom scripts"; exit 1; }
- echo "Copy operation completed"
- chmod 644 $out/etc/postgresql-custom/supautils.conf
- chmod 644 $out/etc/postgresql/postgresql.conf
- chmod 644 $out/etc/postgresql-custom/logging.conf
- chmod 644 $out/etc/postgresql/pg_hba.conf
-
- substitute ${./nix/tools/run-server.sh.in} $out/bin/start-postgres-server \
- ${builtins.concatStringsSep " " (builtins.attrValues (builtins.mapAttrs
- (name: value: "--subst-var-by '${name}' '${value}'")
- substitutions
- ))}
- chmod +x $out/bin/start-postgres-server
- '';
+ echo "Copy operation completed"
+ chmod 644 $out/etc/postgresql-custom/supautils.conf
+ chmod 644 $out/etc/postgresql/postgresql.conf
+ chmod 644 $out/etc/postgresql-custom/logging.conf
+ chmod 644 $out/etc/postgresql/pg_hba.conf
+
+ substitute ${./nix/tools/run-server.sh.in} $out/bin/start-postgres-server \
+ ${builtins.concatStringsSep " " (builtins.attrValues (builtins.mapAttrs
+ (name: value: "--subst-var-by '${name}' '${value}'")
+ substitutions
+ ))}
+ chmod +x $out/bin/start-postgres-server
+ '';
# The base set of packages that we export from this Nix Flake, that can
# be used with 'nix build'. Don't use the names listed below; check the
# name in 'nix flake show' in order to make sure exactly what name you
# want.
- basePackages = let
- # Function to get the PostgreSQL version from the attribute name
- getVersion = name:
- let
- match = builtins.match "psql_([0-9]+)" name;
- in
- if match == null then null else builtins.head match;
-
- # Define the available PostgreSQL versions
- postgresVersions = {
- psql_15 = makePostgres "15";
- psql_17 = makePostgres "17";
- psql_orioledb-17 = makePostgres "orioledb-17" ;
- };
+ basePackages =
+ let
+ # Function to get the PostgreSQL version from the attribute name
+ getVersion = name:
+ let
+ match = builtins.match "psql_([0-9]+)" name;
+ in
+ if match == null then null else builtins.head match;
+
+ # Define the available PostgreSQL versions
+ postgresVersions = {
+ psql_15 = makePostgres "15";
+ psql_17 = makePostgres "17";
+ psql_orioledb-17 = makePostgres "orioledb-17";
+ };
- # Find the active PostgreSQL version
- activeVersion = getVersion (builtins.head (builtins.attrNames postgresVersions));
+ # Find the active PostgreSQL version
+ activeVersion = getVersion (builtins.head (builtins.attrNames postgresVersions));
- # Function to create the pg_regress package
- makePgRegress = version:
- let
- postgresqlPackage = pkgs."postgresql_${version}";
- in
- pkgs.callPackage ./nix/ext/pg_regress.nix {
+ # Function to create the pg_regress package
+ makePgRegress = version:
+ let
+ postgresqlPackage = pkgs."postgresql_${version}";
+ in
+ pkgs.callPackage ./nix/ext/pg_regress.nix {
postgresql = postgresqlPackage;
};
- postgresql_15 = getPostgresqlPackage "15";
- postgresql_17 = getPostgresqlPackage "17";
- postgresql_orioledb-17 = getPostgresqlPackage "orioledb-17";
- in
- postgresVersions // {
- supabase-groonga = supabase-groonga;
- cargo-pgrx_0_11_3 = pkgs.cargo-pgrx.cargo-pgrx_0_11_3;
- cargo-pgrx_0_12_6 = pkgs.cargo-pgrx.cargo-pgrx_0_12_6;
- cargo-pgrx_0_12_9 = pkgs.cargo-pgrx.cargo-pgrx_0_12_9;
- # PostgreSQL versions.
- psql_15 = postgresVersions.psql_15;
- psql_17 = postgresVersions.psql_17;
- psql_orioledb-17 = postgresVersions.psql_orioledb-17;
- wal-g-2 = wal-g-2;
- wal-g-3 = wal-g-3;
- sfcgal = sfcgal;
- pg_prove = pkgs.perlPackages.TAPParserSourceHandlerpgTAP;
- inherit postgresql_15 postgresql_17 postgresql_orioledb-17;
- postgresql_15_debug = if pkgs.stdenv.isLinux then postgresql_15.debug else null;
- postgresql_17_debug = if pkgs.stdenv.isLinux then postgresql_17.debug else null;
- postgresql_orioledb-17_debug = if pkgs.stdenv.isLinux then postgresql_orioledb-17.debug else null;
- postgresql_15_src = pkgs.stdenv.mkDerivation {
- pname = "postgresql-15-src";
- version = postgresql_15.version;
-
- src = postgresql_15.src;
-
- nativeBuildInputs = [ pkgs.bzip2 ];
-
- phases = [ "unpackPhase" "installPhase" ];
-
- installPhase = ''
- mkdir -p $out
- cp -r . $out
- '';
+ postgresql_15 = getPostgresqlPackage "15";
+ postgresql_17 = getPostgresqlPackage "17";
+ postgresql_orioledb-17 = getPostgresqlPackage "orioledb-17";
+ in
+ postgresVersions // {
+ supabase-groonga = supabase-groonga;
+ cargo-pgrx_0_11_3 = pkgs.cargo-pgrx.cargo-pgrx_0_11_3;
+ cargo-pgrx_0_12_6 = pkgs.cargo-pgrx.cargo-pgrx_0_12_6;
+ cargo-pgrx_0_12_9 = pkgs.cargo-pgrx.cargo-pgrx_0_12_9;
+ # PostgreSQL versions.
+ psql_15 = postgresVersions.psql_15;
+ psql_17 = postgresVersions.psql_17;
+ psql_orioledb-17 = postgresVersions.psql_orioledb-17;
+ wal-g-2 = wal-g-2;
+ wal-g-3 = wal-g-3;
+ sfcgal = sfcgal;
+ pg_prove = pkgs.perlPackages.TAPParserSourceHandlerpgTAP;
+ inherit postgresql_15 postgresql_17 postgresql_orioledb-17;
+ postgresql_15_debug = if pkgs.stdenv.isLinux then postgresql_15.debug else null;
+ postgresql_17_debug = if pkgs.stdenv.isLinux then postgresql_17.debug else null;
+ postgresql_orioledb-17_debug = if pkgs.stdenv.isLinux then postgresql_orioledb-17.debug else null;
+ postgresql_15_src = pkgs.stdenv.mkDerivation {
+ pname = "postgresql-15-src";
+ version = postgresql_15.version;
+
+ src = postgresql_15.src;
+
+ nativeBuildInputs = [ pkgs.bzip2 ];
+
+ phases = [ "unpackPhase" "installPhase" ];
- meta = with pkgs.lib; {
- description = "PostgreSQL 15 source files";
- homepage = "https://www.postgresql.org/";
- license = licenses.postgresql;
- platforms = platforms.all;
+ installPhase = ''
+ mkdir -p $out
+ cp -r . $out
+ '';
+
+ meta = with pkgs.lib; {
+ description = "PostgreSQL 15 source files";
+ homepage = "https://www.postgresql.org/";
+ license = licenses.postgresql;
+ platforms = platforms.all;
+ };
};
- };
- postgresql_17_src = pkgs.stdenv.mkDerivation {
- pname = "postgresql-17-src";
- version = postgresql_17.version;
- src = postgresql_17.src;
+ postgresql_17_src = pkgs.stdenv.mkDerivation {
+ pname = "postgresql-17-src";
+ version = postgresql_17.version;
+ src = postgresql_17.src;
- nativeBuildInputs = [ pkgs.bzip2 ];
+ nativeBuildInputs = [ pkgs.bzip2 ];
- phases = [ "unpackPhase" "installPhase" ];
+ phases = [ "unpackPhase" "installPhase" ];
- installPhase = ''
- mkdir -p $out
- cp -r . $out
- '';
- meta = with pkgs.lib; {
- description = "PostgreSQL 17 source files";
- homepage = "https://www.postgresql.org/";
- license = licenses.postgresql;
- platforms = platforms.all;
+ installPhase = ''
+ mkdir -p $out
+ cp -r . $out
+ '';
+ meta = with pkgs.lib; {
+ description = "PostgreSQL 17 source files";
+ homepage = "https://www.postgresql.org/";
+ license = licenses.postgresql;
+ platforms = platforms.all;
+ };
};
- };
- postgresql_orioledb-17_src = pkgs.stdenv.mkDerivation {
- pname = "postgresql-17-src";
- version = postgresql_orioledb-17.version;
+ postgresql_orioledb-17_src = pkgs.stdenv.mkDerivation {
+ pname = "postgresql-17-src";
+ version = postgresql_orioledb-17.version;
- src = postgresql_orioledb-17.src;
+ src = postgresql_orioledb-17.src;
- nativeBuildInputs = [ pkgs.bzip2 ];
+ nativeBuildInputs = [ pkgs.bzip2 ];
- phases = [ "unpackPhase" "installPhase" ];
+ phases = [ "unpackPhase" "installPhase" ];
- installPhase = ''
- mkdir -p $out
- cp -r . $out
- '';
+ installPhase = ''
+ mkdir -p $out
+ cp -r . $out
+ '';
- meta = with pkgs.lib; {
- description = "PostgreSQL 15 source files";
- homepage = "https://www.postgresql.org/";
- license = licenses.postgresql;
- platforms = platforms.all;
+ meta = with pkgs.lib; {
+ description = "PostgreSQL 15 source files";
+ homepage = "https://www.postgresql.org/";
+ license = licenses.postgresql;
+ platforms = platforms.all;
+ };
};
- };
- mecab_naist_jdic = mecab-naist-jdic;
- supabase_groonga = supabase-groonga;
- pg_regress = makePgRegress activeVersion;
- # Start a version of the server.
- start-server = makePostgresDevSetup {
- inherit pkgs;
- name = "start-postgres-server";
- };
+ mecab_naist_jdic = mecab-naist-jdic;
+ supabase_groonga = supabase-groonga;
+ pg_regress = makePgRegress activeVersion;
+ # Start a version of the server.
+ start-server = makePostgresDevSetup {
+ inherit pkgs;
+ name = "start-postgres-server";
+ };
+
+ # Start a version of the client and runs migrations script on server.
+ start-client =
+ let
+ migrationsDir = ./migrations/db;
+ postgresqlSchemaSql = ./nix/tools/postgresql_schema.sql;
+ pgbouncerAuthSchemaSql = ./ansible/files/pgbouncer_config/pgbouncer_auth_schema.sql;
+ statExtensionSql = ./ansible/files/stat_extension.sql;
+ in
+ pkgs.runCommand "start-postgres-client" { } ''
+ mkdir -p $out/bin
+ substitute ${./nix/tools/run-client.sh.in} $out/bin/start-postgres-client \
+ --subst-var-by 'PGSQL_DEFAULT_PORT' '${pgsqlDefaultPort}' \
+ --subst-var-by 'PGSQL_SUPERUSER' '${pgsqlSuperuser}' \
+ --subst-var-by 'PSQL15_BINDIR' '${basePackages.psql_15.bin}' \
+ --subst-var-by 'PSQL17_BINDIR' '${basePackages.psql_17.bin}' \
+ --subst-var-by 'PSQLORIOLEDB17_BINDIR' '${basePackages.psql_orioledb-17.bin}' \
+ --subst-var-by 'MIGRATIONS_DIR' '${migrationsDir}' \
+ --subst-var-by 'POSTGRESQL_SCHEMA_SQL' '${postgresqlSchemaSql}' \
+ --subst-var-by 'PGBOUNCER_AUTH_SCHEMA_SQL' '${pgbouncerAuthSchemaSql}' \
+ --subst-var-by 'STAT_EXTENSION_SQL' '${statExtensionSql}'
+ chmod +x $out/bin/start-postgres-client
+ '';
+
+ # Migrate between two data directories.
+ migrate-tool =
+ let
+ configFile = ./nix/tests/postgresql.conf.in;
+ getkeyScript = ./nix/tests/util/pgsodium_getkey.sh;
+ primingScript = ./nix/tests/prime.sql;
+ migrationData = ./nix/tests/migrations/data.sql;
+ in
+ pkgs.runCommand "migrate-postgres" { } ''
+ mkdir -p $out/bin
+ substitute ${./nix/tools/migrate-tool.sh.in} $out/bin/migrate-postgres \
+ --subst-var-by 'PSQL15_BINDIR' '${basePackages.psql_15.bin}' \
+ --subst-var-by 'PSQL_CONF_FILE' '${configFile}' \
+ --subst-var-by 'PGSODIUM_GETKEY' '${getkeyScript}' \
+ --subst-var-by 'PRIMING_SCRIPT' '${primingScript}' \
+ --subst-var-by 'MIGRATION_DATA' '${migrationData}'
+
+ chmod +x $out/bin/migrate-postgres
+ '';
- # Start a version of the client and runs migrations script on server.
- start-client =
- let
- migrationsDir = ./migrations/db;
- postgresqlSchemaSql = ./nix/tools/postgresql_schema.sql;
- pgbouncerAuthSchemaSql = ./ansible/files/pgbouncer_config/pgbouncer_auth_schema.sql;
- statExtensionSql = ./ansible/files/stat_extension.sql;
- in
- pkgs.runCommand "start-postgres-client" { } ''
+ start-replica = pkgs.runCommand "start-postgres-replica" { } ''
mkdir -p $out/bin
- substitute ${./nix/tools/run-client.sh.in} $out/bin/start-postgres-client \
- --subst-var-by 'PGSQL_DEFAULT_PORT' '${pgsqlDefaultPort}' \
+ substitute ${./nix/tools/run-replica.sh.in} $out/bin/start-postgres-replica \
--subst-var-by 'PGSQL_SUPERUSER' '${pgsqlSuperuser}' \
- --subst-var-by 'PSQL15_BINDIR' '${basePackages.psql_15.bin}' \
- --subst-var-by 'PSQL17_BINDIR' '${basePackages.psql_17.bin}' \
- --subst-var-by 'PSQLORIOLEDB17_BINDIR' '${basePackages.psql_orioledb-17.bin}' \
- --subst-var-by 'MIGRATIONS_DIR' '${migrationsDir}' \
- --subst-var-by 'POSTGRESQL_SCHEMA_SQL' '${postgresqlSchemaSql}' \
- --subst-var-by 'PGBOUNCER_AUTH_SCHEMA_SQL' '${pgbouncerAuthSchemaSql}' \
- --subst-var-by 'STAT_EXTENSION_SQL' '${statExtensionSql}'
- chmod +x $out/bin/start-postgres-client
+ --subst-var-by 'PSQL15_BINDIR' '${basePackages.psql_15.bin}'
+ chmod +x $out/bin/start-postgres-replica
+ '';
+ pg-restore =
+ pkgs.runCommand "run-pg-restore" { } ''
+ mkdir -p $out/bin
+ substitute ${./nix/tools/run-restore.sh.in} $out/bin/pg-restore \
+ --subst-var-by PSQL15_BINDIR '${basePackages.psql_15.bin}'
+ chmod +x $out/bin/pg-restore
+ '';
+ sync-exts-versions = pkgs.runCommand "sync-exts-versions" { } ''
+ mkdir -p $out/bin
+ substitute ${./nix/tools/sync-exts-versions.sh.in} $out/bin/sync-exts-versions \
+ --subst-var-by 'YQ' '${pkgs.yq}/bin/yq' \
+ --subst-var-by 'JQ' '${pkgs.jq}/bin/jq' \
+ --subst-var-by 'NIX_EDITOR' '${nix-editor.packages.${system}.nix-editor}/bin/nix-editor' \
+ --subst-var-by 'NIXPREFETCHURL' '${pkgs.nixVersions.nix_2_20}/bin/nix-prefetch-url' \
+ --subst-var-by 'NIX' '${pkgs.nixVersions.nix_2_20}/bin/nix'
+ chmod +x $out/bin/sync-exts-versions
'';
- # Migrate between two data directories.
- migrate-tool =
- let
- configFile = ./nix/tests/postgresql.conf.in;
- getkeyScript = ./nix/tests/util/pgsodium_getkey.sh;
- primingScript = ./nix/tests/prime.sql;
- migrationData = ./nix/tests/migrations/data.sql;
- in
- pkgs.runCommand "migrate-postgres" { } ''
+ local-infra-bootstrap = pkgs.runCommand "local-infra-bootstrap" { } ''
mkdir -p $out/bin
- substitute ${./nix/tools/migrate-tool.sh.in} $out/bin/migrate-postgres \
- --subst-var-by 'PSQL15_BINDIR' '${basePackages.psql_15.bin}' \
- --subst-var-by 'PSQL_CONF_FILE' '${configFile}' \
- --subst-var-by 'PGSODIUM_GETKEY' '${getkeyScript}' \
- --subst-var-by 'PRIMING_SCRIPT' '${primingScript}' \
- --subst-var-by 'MIGRATION_DATA' '${migrationData}'
-
- chmod +x $out/bin/migrate-postgres
+ substitute ${./nix/tools/local-infra-bootstrap.sh.in} $out/bin/local-infra-bootstrap
+ chmod +x $out/bin/local-infra-bootstrap
'';
-
- start-replica = pkgs.runCommand "start-postgres-replica" { } ''
- mkdir -p $out/bin
- substitute ${./nix/tools/run-replica.sh.in} $out/bin/start-postgres-replica \
- --subst-var-by 'PGSQL_SUPERUSER' '${pgsqlSuperuser}' \
- --subst-var-by 'PSQL15_BINDIR' '${basePackages.psql_15.bin}'
- chmod +x $out/bin/start-postgres-replica
- '';
- pg-restore =
- pkgs.runCommand "run-pg-restore" { } ''
+ dbmate-tool =
+ let
+ migrationsDir = ./migrations/db;
+ ansibleVars = ./ansible/vars.yml;
+ pgbouncerAuthSchemaSql = ./ansible/files/pgbouncer_config/pgbouncer_auth_schema.sql;
+ statExtensionSql = ./ansible/files/stat_extension.sql;
+ in
+ pkgs.runCommand "dbmate-tool"
+ {
+ buildInputs = with pkgs; [
+ overmind
+ dbmate
+ nix
+ jq
+ yq
+ ];
+ nativeBuildInputs = with pkgs; [
+ makeWrapper
+ ];
+ } ''
+ mkdir -p $out/bin $out/migrations
+ cp -r ${migrationsDir}/* $out
+ substitute ${./nix/tools/dbmate-tool.sh.in} $out/bin/dbmate-tool \
+ --subst-var-by 'PGSQL_DEFAULT_PORT' '${pgsqlDefaultPort}' \
+ --subst-var-by 'MIGRATIONS_DIR' $out \
+ --subst-var-by 'PGSQL_SUPERUSER' '${pgsqlSuperuser}' \
+ --subst-var-by 'ANSIBLE_VARS' ${ansibleVars} \
+ --subst-var-by 'CURRENT_SYSTEM' '${system}' \
+ --subst-var-by 'PGBOUNCER_AUTH_SCHEMA_SQL' '${pgbouncerAuthSchemaSql}' \
+ --subst-var-by 'STAT_EXTENSION_SQL' '${statExtensionSql}'
+ chmod +x $out/bin/dbmate-tool
+ wrapProgram $out/bin/dbmate-tool \
+ --prefix PATH : ${pkgs.lib.makeBinPath [ pkgs.overmind pkgs.dbmate pkgs.nix pkgs.jq pkgs.yq ]}
+ '';
+ show-commands = pkgs.runCommand "show-commands"
+ {
+ nativeBuildInputs = [ pkgs.makeWrapper ];
+ buildInputs = [ pkgs.nushell ];
+ } ''
mkdir -p $out/bin
- substitute ${./nix/tools/run-restore.sh.in} $out/bin/pg-restore \
- --subst-var-by PSQL15_BINDIR '${basePackages.psql_15.bin}'
- chmod +x $out/bin/pg-restore
+ cat > $out/bin/show-commands << 'EOF'
+ #!${pkgs.nushell}/bin/nu
+ let json_output = (nix flake show --json --quiet --all-systems | from json)
+ let apps = ($json_output | get apps.${system})
+ $apps | transpose name info | select name | each { |it| echo $"Run this app with: nix run .#($it.name)" }
+ EOF
+ chmod +x $out/bin/show-commands
+ wrapProgram $out/bin/show-commands \
+ --prefix PATH : ${pkgs.nushell}/bin
'';
- sync-exts-versions = pkgs.runCommand "sync-exts-versions" { } ''
- mkdir -p $out/bin
- substitute ${./nix/tools/sync-exts-versions.sh.in} $out/bin/sync-exts-versions \
- --subst-var-by 'YQ' '${pkgs.yq}/bin/yq' \
- --subst-var-by 'JQ' '${pkgs.jq}/bin/jq' \
- --subst-var-by 'NIX_EDITOR' '${nix-editor.packages.${system}.nix-editor}/bin/nix-editor' \
- --subst-var-by 'NIXPREFETCHURL' '${pkgs.nixVersions.nix_2_20}/bin/nix-prefetch-url' \
- --subst-var-by 'NIX' '${pkgs.nixVersions.nix_2_20}/bin/nix'
- chmod +x $out/bin/sync-exts-versions
- '';
-
- local-infra-bootstrap = pkgs.runCommand "local-infra-bootstrap" { } ''
- mkdir -p $out/bin
- substitute ${./nix/tools/local-infra-bootstrap.sh.in} $out/bin/local-infra-bootstrap
- chmod +x $out/bin/local-infra-bootstrap
- '';
- dbmate-tool =
- let
- migrationsDir = ./migrations/db;
- ansibleVars = ./ansible/vars.yml;
- pgbouncerAuthSchemaSql = ./ansible/files/pgbouncer_config/pgbouncer_auth_schema.sql;
- statExtensionSql = ./ansible/files/stat_extension.sql;
- in
- pkgs.runCommand "dbmate-tool" {
- buildInputs = with pkgs; [
- overmind
- dbmate
- nix
- jq
- yq
- ];
- nativeBuildInputs = with pkgs; [
- makeWrapper
- ];
- } ''
- mkdir -p $out/bin $out/migrations
- cp -r ${migrationsDir}/* $out
- substitute ${./nix/tools/dbmate-tool.sh.in} $out/bin/dbmate-tool \
- --subst-var-by 'PGSQL_DEFAULT_PORT' '${pgsqlDefaultPort}' \
- --subst-var-by 'MIGRATIONS_DIR' $out \
- --subst-var-by 'PGSQL_SUPERUSER' '${pgsqlSuperuser}' \
- --subst-var-by 'ANSIBLE_VARS' ${ansibleVars} \
- --subst-var-by 'CURRENT_SYSTEM' '${system}' \
- --subst-var-by 'PGBOUNCER_AUTH_SCHEMA_SQL' '${pgbouncerAuthSchemaSql}' \
- --subst-var-by 'STAT_EXTENSION_SQL' '${statExtensionSql}'
- chmod +x $out/bin/dbmate-tool
- wrapProgram $out/bin/dbmate-tool \
- --prefix PATH : ${pkgs.lib.makeBinPath [ pkgs.overmind pkgs.dbmate pkgs.nix pkgs.jq pkgs.yq ]}
+ update-readme = pkgs.runCommand "update-readme"
+ {
+ nativeBuildInputs = [ pkgs.makeWrapper ];
+ buildInputs = [ pkgs.nushell ];
+ } ''
+ mkdir -p $out/bin
+ cp ${./nix/tools/update_readme.nu} $out/bin/update-readme
+ chmod +x $out/bin/update-readme
+ wrapProgram $out/bin/update-readme \
+ --prefix PATH : ${pkgs.nushell}/bin
'';
- show-commands = pkgs.runCommand "show-commands" {
- nativeBuildInputs = [ pkgs.makeWrapper ];
- buildInputs = [ pkgs.nushell ];
- } ''
- mkdir -p $out/bin
- cat > $out/bin/show-commands << 'EOF'
- #!${pkgs.nushell}/bin/nu
- let json_output = (nix flake show --json --quiet --all-systems | from json)
- let apps = ($json_output | get apps.${system})
- $apps | transpose name info | select name | each { |it| echo $"Run this app with: nix run .#($it.name)" }
- EOF
- chmod +x $out/bin/show-commands
- wrapProgram $out/bin/show-commands \
- --prefix PATH : ${pkgs.nushell}/bin
- '';
- update-readme = pkgs.runCommand "update-readme" {
- nativeBuildInputs = [ pkgs.makeWrapper ];
- buildInputs = [ pkgs.nushell ];
- } ''
- mkdir -p $out/bin
- cp ${./nix/tools/update_readme.nu} $out/bin/update-readme
- chmod +x $out/bin/update-readme
- wrapProgram $out/bin/update-readme \
- --prefix PATH : ${pkgs.nushell}/bin
- '';
- };
+ # Script to run the AMI build and tests locally
+ build-test-ami = pkgs.runCommand "build-test-ami"
+ {
+ buildInputs = with pkgs; [
+ packer
+ awscli2
+ yq
+ jq
+ openssl
+ git
+ coreutils
+ aws-vault
+ ];
+ } ''
+ mkdir -p $out/bin
+ cat > $out/bin/build-test-ami << 'EOL'
+ #!/usr/bin/env bash
+ set -euo pipefail
+
+ show_help() {
+ cat << EOF
+ Usage: build-test-ami [--help] <postgres-version>
+
+ Build AMI images for PostgreSQL testing.
+
+ This script will:
+ 1. Check for required tools and AWS authentication
+ 2. Build two AMI stages using Packer
+ 3. Clean up any temporary instances
+ 4. Output the final AMI name for use with run-testinfra
+
+ Arguments:
+ postgres-version PostgreSQL major version to build (required)
+
+ Options:
+ --help Show this help message and exit
+
+ Requirements:
+ - AWS Vault profile must be set in AWS_VAULT environment variable
+ - Packer, AWS CLI, yq, jq, and OpenSSL must be installed
+ - Must be run from a git repository
+
+ Example:
+ aws-vault exec <profile-name> -- nix run .#build-test-ami 15
+ EOF
+ }
+
+ # Handle help flag
+ if [[ "$#" -gt 0 && "$1" == "--help" ]]; then
+ show_help
+ exit 0
+ fi
+
+ export PATH="${pkgs.lib.makeBinPath (with pkgs; [
+ packer
+ awscli2
+ yq
+ jq
+ openssl
+ git
+ coreutils
+ aws-vault
+ ])}:$PATH"
+
+ # Check for required tools
+ for cmd in packer aws-vault yq jq openssl; do
+ if ! command -v $cmd &> /dev/null; then
+ echo "Error: $cmd is required but not found"
+ exit 1
+ fi
+ done
+
+ # Check AWS Vault profile
+ if [ -z "''${AWS_VAULT:-}" ]; then
+ echo "Error: AWS_VAULT environment variable must be set with the profile name"
+ echo "Usage: aws-vault exec <profile-name> -- nix run .#build-test-ami <postgres-version>"
+ exit 1
+ fi
+
+ # Set values
+ REGION="ap-southeast-1"
+ POSTGRES_VERSION="$1"
+ RANDOM_STRING=$(openssl rand -hex 8)
+ GIT_SHA=$(git rev-parse HEAD)
+ RUN_ID=$(date +%s)
+
+ # Generate common-nix.vars.pkr.hcl
+ PG_VERSION=$(yq -r ".postgres_release[\"postgres$POSTGRES_VERSION\"]" ansible/vars.yml)
+ echo "postgres-version = \"$PG_VERSION\"" > common-nix.vars.pkr.hcl
+
+ # Build AMI Stage 1
+ packer init amazon-arm64-nix.pkr.hcl
+ packer build \
+ -var "git-head-version=$GIT_SHA" \
+ -var "packer-execution-id=$RUN_ID" \
+ -var-file="development-arm.vars.pkr.hcl" \
+ -var-file="common-nix.vars.pkr.hcl" \
+ -var "ansible_arguments=" \
+ -var "postgres-version=$RANDOM_STRING" \
+ -var "region=$REGION" \
+ -var 'ami_regions=["'"$REGION"'"]' \
+ -var "force-deregister=true" \
+ -var "ansible_arguments=-e postgresql_major=$POSTGRES_VERSION" \
+ amazon-arm64-nix.pkr.hcl
+
+ # Build AMI Stage 2
+ packer init stage2-nix-psql.pkr.hcl
+ packer build \
+ -var "git-head-version=$GIT_SHA" \
+ -var "packer-execution-id=$RUN_ID" \
+ -var "postgres_major_version=$POSTGRES_VERSION" \
+ -var-file="development-arm.vars.pkr.hcl" \
+ -var-file="common-nix.vars.pkr.hcl" \
+ -var "postgres-version=$RANDOM_STRING" \
+ -var "region=$REGION" \
+ -var 'ami_regions=["'"$REGION"'"]' \
+ -var "force-deregister=true" \
+ -var "git_sha=$GIT_SHA" \
+ stage2-nix-psql.pkr.hcl
+
+ # Cleanup instances from AMI builds
+ cleanup_instances() {
+ echo "Terminating EC2 instances with tag testinfra-run-id=$RUN_ID..."
+ aws ec2 --region $REGION describe-instances \
+ --filters "Name=tag:testinfra-run-id,Values=$RUN_ID" \
+ --query "Reservations[].Instances[].InstanceId" \
+ --output text | xargs -r aws ec2 terminate-instances \
+ --region $REGION --instance-ids || true
+ }
+
+ # Set up traps for various signals to ensure cleanup
+ trap cleanup_instances EXIT HUP INT QUIT TERM
+
+ # Create and activate virtual environment
+ VENV_DIR=$(mktemp -d)
+ trap 'rm -rf "$VENV_DIR"' EXIT HUP INT QUIT TERM
+ python3 -m venv "$VENV_DIR"
+ source "$VENV_DIR/bin/activate"
+
+ # Install required Python packages
+ echo "Installing required Python packages..."
+ pip install boto3 boto3-stubs[essential] docker ec2instanceconnectcli pytest paramiko requests
+
+ # Run the tests with aws-vault
+ echo "Running tests for AMI: $RANDOM_STRING using AWS Vault profile: $AWS_VAULT_PROFILE"
+ aws-vault exec $AWS_VAULT_PROFILE -- pytest -vv -s testinfra/test_ami_nix.py
+
+ # Deactivate virtual environment (cleanup is handled by trap)
+ deactivate
+ EOL
+ chmod +x $out/bin/build-test-ami
+ '';
+
+ run-testinfra = pkgs.runCommand "run-testinfra"
+ {
+ buildInputs = with pkgs; [
+ aws-vault
+ python3
+ python3Packages.pip
+ coreutils
+ ];
+ } ''
+ mkdir -p $out/bin
+ cat > $out/bin/run-testinfra << 'EOL'
+ #!/usr/bin/env bash
+ set -euo pipefail
+
+ show_help() {
+ cat << EOF
+ Usage: run-testinfra --ami-name NAME [--aws-vault-profile PROFILE]
+
+ Run the testinfra tests locally against a specific AMI.
+
+ This script will:
+ 1. Check if aws-vault is installed and configured
+ 2. Set up the required environment variables
+ 3. Create and activate a virtual environment
+ 4. Install required Python packages from pip
+ 5. Run the tests with aws-vault credentials
+ 6. Clean up the virtual environment
+
+ Required flags:
+ --ami-name NAME The name of the AMI to test
+
+ Optional flags:
+ --aws-vault-profile PROFILE AWS Vault profile to use (default: staging)
+ --help Show this help message and exit
+
+ Requirements:
+ - aws-vault installed and configured
+ - Python 3 with pip
+ - Must be run from the repository root
+
+ Examples:
+ run-testinfra --ami-name supabase-postgres-abc123
+ run-testinfra --ami-name supabase-postgres-abc123 --aws-vault-profile production
+ EOF
+ }
+
+ # Default values
+ AWS_VAULT_PROFILE="staging"
+ AMI_NAME=""
+
+ # Parse arguments
+ while [[ $# -gt 0 ]]; do
+ case $1 in
+ --aws-vault-profile)
+ AWS_VAULT_PROFILE="$2"
+ shift 2
+ ;;
+ --ami-name)
+ AMI_NAME="$2"
+ shift 2
+ ;;
+ --help)
+ show_help
+ exit 0
+ ;;
+ *)
+ echo "Error: Unexpected argument: $1"
+ show_help
+ exit 1
+ ;;
+ esac
+ done
+
+ # Check for required tools
+ if ! command -v aws-vault &> /dev/null; then
+ echo "Error: aws-vault is required but not found"
+ exit 1
+ fi
+
+ # Check for AMI name argument
+ if [ -z "$AMI_NAME" ]; then
+ echo "Error: --ami-name is required"
+ show_help
+ exit 1
+ fi
+
+ # Set environment variables
+ export AWS_REGION="ap-southeast-1"
+ export AWS_DEFAULT_REGION="ap-southeast-1"
+ export AMI_NAME="$AMI_NAME" # Export AMI_NAME for pytest
+ export RUN_ID="local-$(date +%s)" # Generate a unique RUN_ID
+
+ # Function to terminate EC2 instances
+ terminate_instances() {
+ echo "Terminating EC2 instances with tag testinfra-run-id=$RUN_ID..."
+ aws-vault exec $AWS_VAULT_PROFILE -- aws ec2 --region ap-southeast-1 describe-instances \
+ --filters "Name=tag:testinfra-run-id,Values=$RUN_ID" \
+ --query "Reservations[].Instances[].InstanceId" \
+ --output text | xargs -r aws-vault exec $AWS_VAULT_PROFILE -- aws ec2 terminate-instances \
+ --region ap-southeast-1 --instance-ids || true
+ }
+
+ # Set up traps for various signals to ensure cleanup
+ trap terminate_instances EXIT HUP INT QUIT TERM
+
+ # Create and activate virtual environment
+ VENV_DIR=$(mktemp -d)
+ trap 'rm -rf "$VENV_DIR"' EXIT HUP INT QUIT TERM
+ python3 -m venv "$VENV_DIR"
+ source "$VENV_DIR/bin/activate"
+
+ # Install required Python packages
+ echo "Installing required Python packages..."
+ pip install boto3 boto3-stubs[essential] docker ec2instanceconnectcli pytest paramiko requests
+
+ # Function to run tests and ensure cleanup
+ run_tests() {
+ local exit_code=0
+ echo "Running tests for AMI: $AMI_NAME using AWS Vault profile: $AWS_VAULT_PROFILE"
+ aws-vault exec "$AWS_VAULT_PROFILE" -- pytest -vv -s testinfra/test_ami_nix.py || exit_code=$?
+ return $exit_code
+ }
+
+ # Run tests and capture exit code
+ run_tests
+ test_exit_code=$?
+
+ # Deactivate virtual environment
+ deactivate
+
+ # Explicitly call cleanup
+ terminate_instances
+
+ # Exit with the test exit code
+ exit $test_exit_code
+ EOL
+ chmod +x $out/bin/run-testinfra
+ '';
+
+ cleanup-ami = pkgs.runCommand "cleanup-ami"
+ {
+ buildInputs = with pkgs; [
+ awscli2
+ aws-vault
+ ];
+ } ''
+ mkdir -p $out/bin
+ cat > $out/bin/cleanup-ami << 'EOL'
+ #!/usr/bin/env bash
+ set -euo pipefail
+
+ export PATH="${pkgs.lib.makeBinPath (with pkgs; [
+ awscli2
+ aws-vault
+ ])}:$PATH"
+
+ # Check for required tools
+ for cmd in aws-vault; do
+ if ! command -v $cmd &> /dev/null; then
+ echo "Error: $cmd is required but not found"
+ exit 1
+ fi
+ done
+
+ # Check AWS Vault profile
+ if [ -z "''${AWS_VAULT:-}" ]; then
+ echo "Error: AWS_VAULT environment variable must be set with the profile name"
+ echo "Usage: aws-vault exec <profile-name> -- nix run .#cleanup-ami <ami-name>"
+ exit 1
+ fi
+
+ # Check for AMI name argument
+ if [ -z "''${1:-}" ]; then
+ echo "Error: AMI name must be provided"
+ echo "Usage: aws-vault exec <profile-name> -- nix run .#cleanup-ami <ami-name>"
+ exit 1
+ fi
+
+ AMI_NAME="$1"
+ REGION="ap-southeast-1"
+
+ # Deregister AMIs
+ for AMI_PATTERN in "supabase-postgres-ci-ami-test-stage-1" "$AMI_NAME"; do
+ aws ec2 describe-images --region $REGION --owners self \
+ --filters "Name=name,Values=$AMI_PATTERN" \
+ --query 'Images[*].ImageId' --output text | while read -r ami_id; do
+ echo "Deregistering AMI: $ami_id"
+ aws ec2 deregister-image --region $REGION --image-id "$ami_id" || true
+ done
+ done
+ EOL
+ chmod +x $out/bin/cleanup-ami
+ '';
+
+ trigger-nix-build = pkgs.runCommand "trigger-nix-build"
+ {
+ buildInputs = with pkgs; [
+ gh
+ git
+ coreutils
+ ];
+ } ''
+ mkdir -p $out/bin
+ cat > $out/bin/trigger-nix-build << 'EOL'
+ #!/usr/bin/env bash
+ set -euo pipefail
+
+ show_help() {
+ cat << EOF
+ Usage: trigger-nix-build [--help]
+
+ Trigger the nix-build workflow for the current branch and watch its progress.
+
+ This script will:
+ 1. Check if you're authenticated with GitHub
+ 2. Get the current branch and commit
+ 3. Verify you're on a standard branch (develop or release/*) or prompt for confirmation
+ 4. Trigger the nix-build workflow
+ 5. Watch the workflow progress until completion
+
+ Options:
+ --help Show this help message and exit
+
+ Requirements:
+ - GitHub CLI (gh) installed and authenticated
+ - Git installed
+ - Must be run from a git repository
+
+ Example:
+ trigger-nix-build
+ EOF
+ }
+
+ # Handle help flag
+ if [[ "$#" -gt 0 && "$1" == "--help" ]]; then
+ show_help
+ exit 0
+ fi
+
+ export PATH="${pkgs.lib.makeBinPath (with pkgs; [
+ gh
+ git
+ coreutils
+ ])}:$PATH"
+
+ # Check for required tools
+ for cmd in gh git; do
+ if ! command -v $cmd &> /dev/null; then
+ echo "Error: $cmd is required but not found"
+ exit 1
+ fi
+ done
+
+ # Check if user is authenticated with GitHub
+ if ! gh auth status &>/dev/null; then
+ echo "Error: Not authenticated with GitHub. Please run 'gh auth login' first."
+ exit 1
+ fi
+
+ # Get current branch and commit
+ BRANCH=$(git rev-parse --abbrev-ref HEAD)
+ COMMIT=$(git rev-parse HEAD)
+
+ # Check if we're on a standard branch
+ if [[ "$BRANCH" != "develop" && ! "$BRANCH" =~ ^release/ ]]; then
+ echo "Warning: Running workflow from non-standard branch: $BRANCH"
+ echo "This is supported for testing purposes."
+ read -p "Continue? [y/N] " -n 1 -r
+ echo
+ if [[ ! $REPLY =~ ^[Yy]$ ]]; then
+ echo "Aborted."
+ exit 1
+ fi
+ fi
+
+ # Trigger the workflow
+ echo "Triggering nix-build workflow for branch $BRANCH (commit: $COMMIT)"
+ gh workflow run nix-build.yml --ref "$BRANCH"
+
+ # Wait for workflow to start and get the run ID
+ echo "Waiting for workflow to start..."
+ sleep 5
+
+ # Get the latest run ID for this workflow
+ RUN_ID=$(gh run list --workflow=nix-build.yml --limit 1 --json databaseId --jq '.[0].databaseId')
+
+ if [ -z "$RUN_ID" ]; then
+ echo "Error: Could not find workflow run ID"
+ exit 1
+ fi
+
+ echo "Watching workflow run $RUN_ID..."
+ echo "The script will automatically exit when the workflow completes."
+ echo "Press Ctrl+C to stop watching (workflow will continue running)"
+ echo "----------------------------------------"
+
+ # Try to watch the run, but handle network errors gracefully
+ while true; do
+ if gh run watch "$RUN_ID" --exit-status; then
+ break
+ else
+ echo "Network error while watching workflow. Retrying in 5 seconds..."
+ echo "You can also check the status manually with: gh run view $RUN_ID"
+ sleep 5
+ fi
+ done
+ EOL
+ chmod +x $out/bin/trigger-nix-build
+ '';
+ };
# Create a testing harness for a PostgreSQL package. This is used for
@@ -685,10 +1166,10 @@
let
name = pkg.version;
in
- if builtins.match "15.*" name != null then "15"
- else if builtins.match "17.*" name != null then "17"
- else if builtins.match "orioledb-17.*" name != null then "orioledb-17"
- else throw "Unsupported PostgreSQL version: ${name}";
+ if builtins.match "15.*" name != null then "15"
+ else if builtins.match "17.*" name != null then "17"
+ else if builtins.match "orioledb-17.*" name != null then "orioledb-17"
+ else throw "Unsupported PostgreSQL version: ${name}";
# Helper function to filter SQL files based on version
filterTestFiles = version: dir:
@@ -697,7 +1178,7 @@
isValidFile = name:
let
isVersionSpecific = builtins.match "z_.*" name != null;
- matchesVersion =
+ matchesVersion =
if isVersionSpecific
then
if version == "orioledb-17"
@@ -712,163 +1193,174 @@
pkgs.lib.filterAttrs (name: _: isValidFile name) files;
# Get the major version for filtering
- majorVersion =
- let
- version = builtins.trace "pgpkg.version is: ${pgpkg.version}" pgpkg.version;
- _ = builtins.trace "Entering majorVersion logic";
- isOrioledbMatch = builtins.match "^17_[0-9]+$" version != null;
- isSeventeenMatch = builtins.match "^17[.][0-9]+$" version != null;
- result =
- if isOrioledbMatch
- then "orioledb-17"
- else if isSeventeenMatch
- then "17"
- else "15";
- in
- builtins.trace "Major version result: ${result}" result; # Trace the result # For "15.8"
+ majorVersion =
+ let
+ version = builtins.trace "pgpkg.version is: ${pgpkg.version}" pgpkg.version;
+ _ = builtins.trace "Entering majorVersion logic";
+ isOrioledbMatch = builtins.match "^17_[0-9]+$" version != null;
+ isSeventeenMatch = builtins.match "^17[.][0-9]+$" version != null;
+ result =
+ if isOrioledbMatch
+ then "orioledb-17"
+ else if isSeventeenMatch
+ then "17"
+ else "15";
+ in
+ builtins.trace "Major version result: ${result}" result; # Trace the result # For "15.8"
# Filter SQL test files
filteredSqlTests = filterTestFiles majorVersion ./nix/tests/sql;
-
+
# Convert filtered tests to a sorted list of basenames (without extension)
- testList = pkgs.lib.mapAttrsToList (name: _:
- builtins.substring 0 (pkgs.lib.stringLength name - 4) name
- ) filteredSqlTests;
+ testList = pkgs.lib.mapAttrsToList
+ (name: _:
+ builtins.substring 0 (pkgs.lib.stringLength name - 4) name
+ )
+ filteredSqlTests;
sortedTestList = builtins.sort (a: b: a < b) testList;
in
pkgs.runCommand "postgres-${pgpkg.version}-check-harness"
{
- nativeBuildInputs = with pkgs; [
- coreutils bash perl pgpkg pg_prove pg_regress procps
- start-postgres-server-bin which getkey-script supabase-groonga
+ nativeBuildInputs = with pkgs; [
+ coreutils
+ bash
+ perl
+ pgpkg
+ pg_prove
+ pg_regress
+ procps
+ start-postgres-server-bin
+ which
+ getkey-script
+ supabase-groonga
];
} ''
- set -e
-
- #First we need to create a generic pg cluster for pgtap tests and run those
- export GRN_PLUGINS_DIR=${supabase-groonga}/lib/groonga/plugins
- PGTAP_CLUSTER=$(mktemp -d)
- initdb --locale=C --username=supabase_admin -D "$PGTAP_CLUSTER"
- substitute ${./nix/tests/postgresql.conf.in} "$PGTAP_CLUSTER"/postgresql.conf \
- --subst-var-by PGSODIUM_GETKEY_SCRIPT "${getkey-script}/bin/pgsodium-getkey"
- echo "listen_addresses = '*'" >> "$PGTAP_CLUSTER"/postgresql.conf
- echo "port = 5435" >> "$PGTAP_CLUSTER"/postgresql.conf
- echo "host all all 127.0.0.1/32 trust" >> $PGTAP_CLUSTER/pg_hba.conf
- echo "Checking shared_preload_libraries setting:"
- grep -rn "shared_preload_libraries" "$PGTAP_CLUSTER"/postgresql.conf
- # Remove timescaledb if running orioledb-17 check
- echo "I AM ${pgpkg.version}===================================================="
- if [[ "${pgpkg.version}" == *"17"* ]]; then
- perl -pi -e 's/ timescaledb,//g' "$PGTAP_CLUSTER/postgresql.conf"
+ set -e
+
+ #First we need to create a generic pg cluster for pgtap tests and run those
+ export GRN_PLUGINS_DIR=${supabase-groonga}/lib/groonga/plugins
+ PGTAP_CLUSTER=$(mktemp -d)
+ initdb --locale=C --username=supabase_admin -D "$PGTAP_CLUSTER"
+ substitute ${./nix/tests/postgresql.conf.in} "$PGTAP_CLUSTER"/postgresql.conf \
+ --subst-var-by PGSODIUM_GETKEY_SCRIPT "${getkey-script}/bin/pgsodium-getkey"
+ echo "listen_addresses = '*'" >> "$PGTAP_CLUSTER"/postgresql.conf
+ echo "port = 5435" >> "$PGTAP_CLUSTER"/postgresql.conf
+ echo "host all all 127.0.0.1/32 trust" >> $PGTAP_CLUSTER/pg_hba.conf
+ echo "Checking shared_preload_libraries setting:"
+ grep -rn "shared_preload_libraries" "$PGTAP_CLUSTER"/postgresql.conf
+ # Remove timescaledb if running orioledb-17 check
+ echo "I AM ${pgpkg.version}===================================================="
+ if [[ "${pgpkg.version}" == *"17"* ]]; then
+ perl -pi -e 's/ timescaledb,//g' "$PGTAP_CLUSTER/postgresql.conf"
+ fi
+ #NOTE in the future we may also need to add the orioledb extension to the cluster when cluster is oriole
+ echo "PGTAP_CLUSTER directory contents:"
+ ls -la "$PGTAP_CLUSTER"
+
+ # Check if postgresql.conf exists
+ if [ ! -f "$PGTAP_CLUSTER/postgresql.conf" ]; then
+ echo "postgresql.conf is missing!"
+ exit 1
+ fi
+
+ # PostgreSQL startup
+ if [[ "$(uname)" == "Darwin" ]]; then
+ pg_ctl -D "$PGTAP_CLUSTER" -l "$PGTAP_CLUSTER"/postgresql.log -o "-k "$PGTAP_CLUSTER" -p 5435 -d 5" start 2>&1
+ else
+ mkdir -p "$PGTAP_CLUSTER/sockets"
+ pg_ctl -D "$PGTAP_CLUSTER" -l "$PGTAP_CLUSTER"/postgresql.log -o "-k $PGTAP_CLUSTER/sockets -p 5435 -d 5" start 2>&1
+ fi || {
+ echo "pg_ctl failed to start PostgreSQL"
+ echo "Contents of postgresql.log:"
+ cat "$PGTAP_CLUSTER"/postgresql.log
+ exit 1
+ }
+ for i in {1..60}; do
+ if pg_isready -h localhost -p 5435; then
+ echo "PostgreSQL is ready"
+ break
fi
- #NOTE in the future we may also need to add the orioledb extension to the cluster when cluster is oriole
- echo "PGTAP_CLUSTER directory contents:"
- ls -la "$PGTAP_CLUSTER"
-
- # Check if postgresql.conf exists
- if [ ! -f "$PGTAP_CLUSTER/postgresql.conf" ]; then
- echo "postgresql.conf is missing!"
- exit 1
+ sleep 1
+ if [ $i -eq 60 ]; then
+ echo "PostgreSQL is not ready after 60 seconds"
+ echo "PostgreSQL status:"
+ pg_ctl -D "$PGTAP_CLUSTER" status
+ echo "PostgreSQL log content:"
+ cat "$PGTAP_CLUSTER"/postgresql.log
+ exit 1
fi
-
- # PostgreSQL startup
- if [[ "$(uname)" == "Darwin" ]]; then
- pg_ctl -D "$PGTAP_CLUSTER" -l "$PGTAP_CLUSTER"/postgresql.log -o "-k "$PGTAP_CLUSTER" -p 5435 -d 5" start 2>&1
- else
- mkdir -p "$PGTAP_CLUSTER/sockets"
- pg_ctl -D "$PGTAP_CLUSTER" -l "$PGTAP_CLUSTER"/postgresql.log -o "-k $PGTAP_CLUSTER/sockets -p 5435 -d 5" start 2>&1
- fi || {
- echo "pg_ctl failed to start PostgreSQL"
- echo "Contents of postgresql.log:"
+ done
+ createdb -p 5435 -h localhost --username=supabase_admin testing
+ if ! psql -p 5435 -h localhost --username=supabase_admin -d testing -v ON_ERROR_STOP=1 -Xaf ${./nix/tests/prime.sql}; then
+ echo "Error executing SQL file. PostgreSQL log content:"
cat "$PGTAP_CLUSTER"/postgresql.log
+ pg_ctl -D "$PGTAP_CLUSTER" stop
exit 1
- }
- for i in {1..60}; do
- if pg_isready -h localhost -p 5435; then
- echo "PostgreSQL is ready"
- break
+ fi
+ SORTED_DIR=$(mktemp -d)
+ for t in $(printf "%s\n" ${builtins.concatStringsSep " " sortedTestList}); do
+ psql -p 5435 -h localhost --username=supabase_admin -d testing -f "${./nix/tests/sql}/$t.sql" || true
+ done
+ rm -rf "$SORTED_DIR"
+ pg_ctl -D "$PGTAP_CLUSTER" stop
+ rm -rf $PGTAP_CLUSTER
+
+ # End of pgtap tests
+ # from here on out we are running pg_regress tests, we use a different cluster for this
+ # which is start by the start-postgres-server-bin script
+ # start-postgres-server-bin script closely matches our AMI setup, configurations and migrations
+
+ # Ensure pgsodium key directory exists with proper permissions
+ if [[ "$(uname)" == "Darwin" ]]; then
+ mkdir -p /private/tmp/pgsodium
+ chmod 1777 /private/tmp/pgsodium
+ fi
+ unset GRN_PLUGINS_DIR
+ ${start-postgres-server-bin}/bin/start-postgres-server ${getVersionArg pgpkg} --daemonize
+
+ for i in {1..60}; do
+ if pg_isready -h localhost -p 5435 -U supabase_admin -q; then
+ echo "PostgreSQL is ready"
+ break
fi
sleep 1
if [ $i -eq 60 ]; then
- echo "PostgreSQL is not ready after 60 seconds"
- echo "PostgreSQL status:"
- pg_ctl -D "$PGTAP_CLUSTER" status
- echo "PostgreSQL log content:"
- cat "$PGTAP_CLUSTER"/postgresql.log
- exit 1
+ echo "PostgreSQL failed to start"
+ exit 1
fi
- done
- createdb -p 5435 -h localhost --username=supabase_admin testing
- if ! psql -p 5435 -h localhost --username=supabase_admin -d testing -v ON_ERROR_STOP=1 -Xaf ${./nix/tests/prime.sql}; then
- echo "Error executing SQL file. PostgreSQL log content:"
- cat "$PGTAP_CLUSTER"/postgresql.log
- pg_ctl -D "$PGTAP_CLUSTER" stop
- exit 1
- fi
- SORTED_DIR=$(mktemp -d)
- for t in $(printf "%s\n" ${builtins.concatStringsSep " " sortedTestList}); do
- psql -p 5435 -h localhost --username=supabase_admin -d testing -f "${./nix/tests/sql}/$t.sql" || true
- done
- rm -rf "$SORTED_DIR"
- pg_ctl -D "$PGTAP_CLUSTER" stop
- rm -rf $PGTAP_CLUSTER
-
- # End of pgtap tests
- # from here on out we are running pg_regress tests, we use a different cluster for this
- # which is start by the start-postgres-server-bin script
- # start-postgres-server-bin script closely matches our AMI setup, configurations and migrations
-
- # Ensure pgsodium key directory exists with proper permissions
- if [[ "$(uname)" == "Darwin" ]]; then
- mkdir -p /private/tmp/pgsodium
- chmod 1777 /private/tmp/pgsodium
- fi
- unset GRN_PLUGINS_DIR
- ${start-postgres-server-bin}/bin/start-postgres-server ${getVersionArg pgpkg} --daemonize
-
- for i in {1..60}; do
- if pg_isready -h localhost -p 5435 -U supabase_admin -q; then
- echo "PostgreSQL is ready"
- break
- fi
- sleep 1
- if [ $i -eq 60 ]; then
- echo "PostgreSQL failed to start"
- exit 1
- fi
- done
-
- if ! psql -p 5435 -h localhost --no-password --username=supabase_admin -d postgres -v ON_ERROR_STOP=1 -Xaf ${./nix/tests/prime.sql}; then
- echo "Error executing SQL file"
- exit 1
- fi
+ done
- mkdir -p $out/regression_output
- if ! pg_regress \
- --use-existing \
- --dbname=postgres \
- --inputdir=${./nix/tests} \
- --outputdir=$out/regression_output \
- --host=localhost \
- --port=5435 \
- --user=supabase_admin \
- ${builtins.concatStringsSep " " sortedTestList}; then
- echo "pg_regress tests failed"
- cat $out/regression_output/regression.diffs
- exit 1
- fi
+ if ! psql -p 5435 -h localhost --no-password --username=supabase_admin -d postgres -v ON_ERROR_STOP=1 -Xaf ${./nix/tests/prime.sql}; then
+ echo "Error executing SQL file"
+ exit 1
+ fi
+
+ mkdir -p $out/regression_output
+ if ! pg_regress \
+ --use-existing \
+ --dbname=postgres \
+ --inputdir=${./nix/tests} \
+ --outputdir=$out/regression_output \
+ --host=localhost \
+ --port=5435 \
+ --user=supabase_admin \
+ ${builtins.concatStringsSep " " sortedTestList}; then
+ echo "pg_regress tests failed"
+ cat $out/regression_output/regression.diffs
+ exit 1
+ fi
- echo "Running migrations tests"
- pg_prove -p 5435 -U supabase_admin -h localhost -d postgres -v ${./migrations/tests}/test.sql
+ echo "Running migrations tests"
+ pg_prove -p 5435 -U supabase_admin -h localhost -d postgres -v ${./migrations/tests}/test.sql
- # Copy logs to output
- for logfile in $(find /tmp -name postgresql.log -type f); do
- cp "$logfile" $out/postgresql.log
- done
- exit 0
- '';
- in
+ # Copy logs to output
+ for logfile in $(find /tmp -name postgresql.log -type f); do
+ cp "$logfile" $out/postgresql.log
+ done
+ exit 0
+ '';
+ in
rec {
# The list of all packages that can be built with 'nix build'. The list
# of names that can be used can be shown with 'nix flake show'
@@ -907,6 +1399,10 @@
dbmate-tool = mkApp "dbmate-tool" "dbmate-tool";
update-readme = mkApp "update-readme" "update-readme";
show-commands = mkApp "show-commands" "show-commands";
+ build-test-ami = mkApp "build-test-ami" "build-test-ami";
+ run-testinfra = mkApp "run-testinfra" "run-testinfra";
+ cleanup-ami = mkApp "cleanup-ami" "cleanup-ami";
+ trigger-nix-build = mkApp "trigger-nix-build" "trigger-nix-build";
};
# 'devShells.default' lists the set of packages that are included in the
@@ -914,54 +1410,59 @@
# for development and puts many convenient devtools instantly within
# reach.
- devShells = let
- mkCargoPgrxDevShell = { pgrxVersion, rustVersion }: pkgs.mkShell {
- packages = with pkgs; [
- basePackages."cargo-pgrx_${pgrxVersion}"
- (rust-bin.stable.${rustVersion}.default.override {
- extensions = [ "rust-src" ];
- })
- ];
- shellHook = ''
- export HISTFILE=.history
- '';
- };
- in {
- default = pkgs.mkShell {
- packages = with pkgs; [
- coreutils
- just
- nix-update
- #pg_prove
- shellcheck
- ansible
- ansible-lint
- (packer.overrideAttrs (oldAttrs: {
- version = "1.7.8";
- }))
-
- basePackages.start-server
- basePackages.start-client
- basePackages.start-replica
- basePackages.migrate-tool
- basePackages.sync-exts-versions
- dbmate
- nushell
- ];
- shellHook = ''
- export HISTFILE=.history
- export DATABASE_URL="postgres://supabase_admin@localhost:5435/postgres?sslmode=disable"
- '';
- };
- cargo-pgrx_0_11_3 = mkCargoPgrxDevShell {
- pgrxVersion = "0_11_3";
- rustVersion = "1.80.0";
- };
- cargo-pgrx_0_12_6 = mkCargoPgrxDevShell {
- pgrxVersion = "0_12_6";
- rustVersion = "1.80.0";
- };
- };
- }
- );
+ devShells =
+ let
+ mkCargoPgrxDevShell = { pgrxVersion, rustVersion }: pkgs.mkShell {
+ packages = with pkgs; [
+ basePackages."cargo-pgrx_${pgrxVersion}"
+ (rust-bin.stable.${rustVersion}.default.override {
+ extensions = [ "rust-src" ];
+ })
+ ];
+ shellHook = ''
+ export HISTFILE=.history
+ '';
+ };
+ in
+ {
+ default = pkgs.mkShell {
+ packages = with pkgs; [
+ coreutils
+ just
+ nix-update
+ #pg_prove
+ shellcheck
+ ansible
+ ansible-lint
+ (packer.overrideAttrs (oldAttrs: {
+ version = "1.7.8";
+ }))
+
+ basePackages.start-server
+ basePackages.start-client
+ basePackages.start-replica
+ basePackages.migrate-tool
+ basePackages.sync-exts-versions
+ basePackages.build-test-ami
+ basePackages.run-testinfra
+ basePackages.cleanup-ami
+ dbmate
+ nushell
+ pythonEnv
+ ];
+ shellHook = ''
+ export HISTFILE=.history
+ '';
+ };
+ cargo-pgrx_0_11_3 = mkCargoPgrxDevShell {
+ pgrxVersion = "0_11_3";
+ rustVersion = "1.80.0";
+ };
+ cargo-pgrx_0_12_6 = mkCargoPgrxDevShell {
+ pgrxVersion = "0_12_6";
+ rustVersion = "1.80.0";
+ };
+ };
+ }
+ );
}
diff --git a/nix/docs/development-workflow.md b/nix/docs/development-workflow.md
new file mode 100644
index 00000000..695427ab
--- /dev/null
+++ b/nix/docs/development-workflow.md
@@ -0,0 +1,141 @@
+# PostgreSQL Development Workflow
+
+This document outlines the workflow for developing and testing PostgreSQL in an ec2 instance using the tools provided in this repo.
+
+## Prerequisites
+
+- Nix installed and configured
+- AWS credentials configured with aws-vault (you must set up aws-vault beforehand)
+- GitHub access to the repository
+
+## Workflow Steps
+
+### 1. Trigger Remote Build and Cache
+
+To build, test, and cache your changes in the Supabase Nix binary cache:
+
+```bash
+# From your branch
+nix run .#trigger-nix-build
+```
+
+This will:
+- Trigger a GitHub Actions workflow
+- Build PostgreSQL and extensions
+- Run nix flake check tests (evaluation of nix code, pg_regress and migrations tests)
+- Cache the results in the Supabase Nix binary cache
+- Watch the workflow progress until completion
+
+The workflow will run on the branch you're currently on.
+
+If you're on a feature different branch, you'll be prompted to confirm before proceeding.
+
+### 2. Build AMI
+
+After the build is complete and cached, build the AMI:
+
+```bash
+# Build AMI for PostgreSQL 15
+aws-vault exec <profile-name> -- nix run .#build-test-ami 15
+
+# Or for PostgreSQL 17
+aws-vault exec <profile-name> -- nix run .#build-test-ami 17
+
+# Or for PostgreSQL orioledb-17
+aws-vault exec <profile-name> -- nix run .#build-test-ami orioledb-17
+```
+
+This will:
+- Build two AMI stages using Packer
+- Clean up temporary instances after AMI builds
+- Output the final AMI name (e.g., `supabase-postgres-abc123`)
+
+**Important**: Take note of the AMI name output at the end, as you'll need it for the next step.
+
+### 3. Run Testinfra
+
+Run the testinfra tests against the AMI:
+
+```bash
+# Run tests against the AMI
+nix run .#run-testinfra -- --aws-vault-profile <profile-name> --ami-name supabase-postgres-abc123
+```
+
+This will:
+- Create a Python virtual environment
+- Install required Python packages
+- Create an EC2 instance from the AMI
+- Run the test suite
+- Automatically terminate the EC2 instance when done
+
+The script handles:
+- Setting up AWS credentials via aws-vault
+- Creating and managing the Python virtual environment
+- Running the tests
+- Cleaning up EC2 instances
+- Proper error handling and cleanup on interruption
+
+### 4. Optional: Cleanup AMI
+
+If you want to clean up the AMI after testing:
+
+```bash
+# Clean up the AMI
+aws-vault exec <profile-name> -- nix run .#cleanup-ami supabase-postgres-abc123
+```
+
+This will:
+- Deregister the AMI
+- Clean up any associated resources
+
+## Troubleshooting
+
+### Common Issues
+
+1. **AWS Credentials**
+ - Ensure aws-vault is properly configured
+ - Use the `--aws-vault-profile` argument to specify your AWS profile
+ - Default profile is "staging" if not specified
+
+2. **EC2 Instance Not Terminating**
+ - The script includes multiple safeguards for cleanup
+ - If instances aren't terminated, check AWS console and terminate manually
+
+3. **Test Failures**
+ - Check the test output for specific failures
+ - Ensure you're using the correct AMI name
+ - Verify AWS region and permissions
+
+### Environment Variables
+
+The following environment variables are used:
+- `AWS_VAULT`: AWS Vault profile name (default: staging)
+- `AWS_REGION`: AWS region (default: ap-southeast-1)
+- `AMI_NAME`: Name of the AMI to test
+
+## Best Practices
+
+1. **Branch Management**
+ - Use feature branches for development
+ - Merge to develop for testing
+ - Use release branches for version-specific changes
+
+2. **Resource Cleanup**
+ - Always run the cleanup step after testing
+ - Monitor AWS console for any lingering resources
+ - Use the cleanup-ami command when done with an AMI
+
+3. **Testing**
+ - Run tests locally before pushing changes
+ - Verify AMI builds before running testinfra
+ - Check test output for any warnings or errors
+
+## Additional Commands
+
+```bash
+# Show available commands
+nix run .#show-commands
+
+# Update README with latest command information
+nix run .#update-readme
+```
\ No newline at end of file
diff --git a/nix/ext/pg_jsonschema.nix b/nix/ext/pg_jsonschema.nix
index 642519f0..654bb93f 100644
--- a/nix/ext/pg_jsonschema.nix
+++ b/nix/ext/pg_jsonschema.nix
@@ -26,7 +26,10 @@ buildPgrxExtension_0_12_6 rec {
env = lib.optionalAttrs stdenv.isDarwin {
POSTGRES_LIB = "${postgresql}/lib";
RUSTFLAGS = "-C link-arg=-undefined -C link-arg=dynamic_lookup";
- PGPORT = "5433";
+ PGPORT = toString (5441 +
+ (if builtins.match ".*_.*" postgresql.version != null then 1 else 0) + # +1 for OrioleDB
+ ((builtins.fromJSON (builtins.substring 0 2 postgresql.version)) - 15) * 2); # +2 for each major version
+
};
cargoLock = {
diff --git a/testinfra/test_ami_nix.py b/testinfra/test_ami_nix.py
index 4d354fac..1975818d 100644
--- a/testinfra/test_ami_nix.py
+++ b/testinfra/test_ami_nix.py
@@ -6,10 +6,11 @@ import os
import pytest
import requests
import socket
-import testinfra
from ec2instanceconnectcli.EC2InstanceConnectLogger import EC2InstanceConnectLogger
from ec2instanceconnectcli.EC2InstanceConnectKey import EC2InstanceConnectKey
from time import sleep
+import subprocess
+import paramiko
# if GITHUB_RUN_ID is not set, use a default value that includes the user and hostname
RUN_ID = os.environ.get(
@@ -170,6 +171,51 @@ logger.addHandler(handler)
logger.setLevel(logging.DEBUG)
+def get_ssh_connection(instance_ip, ssh_identity_file, max_retries=10):
+ """Create and return a single SSH connection that can be reused."""
+ for attempt in range(max_retries):
+ try:
+ # Create SSH client
+ ssh = paramiko.SSHClient()
+ ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+
+ # Connect with our working parameters
+ ssh.connect(
+ hostname=instance_ip,
+ username='ubuntu',
+ key_filename=ssh_identity_file,
+ timeout=10,
+ banner_timeout=10
+ )
+
+ # Test the connection
+ stdin, stdout, stderr = ssh.exec_command('echo "SSH test"')
+ if stdout.channel.recv_exit_status() == 0 and "SSH test" in stdout.read().decode():
+ logger.info("SSH connection established successfully")
+ return ssh
+ else:
+ raise Exception("SSH test command failed")
+
+ except Exception as e:
+ if attempt == max_retries - 1:
+ raise
+ logger.warning(
+ f"Ssh connection failed, retrying: {attempt + 1}/{max_retries} failed, retrying ..."
+ )
+ sleep(5)
+
+
+def run_ssh_command(ssh, command):
+ """Run a command over the established SSH connection."""
+ stdin, stdout, stderr = ssh.exec_command(command)
+ exit_code = stdout.channel.recv_exit_status()
+ return {
+ 'succeeded': exit_code == 0,
+ 'stdout': stdout.read().decode(),
+ 'stderr': stderr.read().decode()
+ }
+
+
# scope='session' uses the same container for all the tests;
# scope='function' uses a new container per test function.
@pytest.fixture(scope="session")
@@ -230,6 +276,7 @@ runcmd:
- 'sudo echo \"pgbouncer\" \"postgres\" >> /etc/pgbouncer/userlist.txt'
- 'cd /tmp && aws s3 cp --region ap-southeast-1 s3://init-scripts-staging/project/init.sh .'
- 'bash init.sh "staging"'
+ - 'touch /var/lib/init-complete'
- 'rm -rf /tmp/*'
""",
TagSpecifications=[
@@ -256,108 +303,95 @@ runcmd:
)
assert response["Success"]
- # instance doesn't have public ip yet
+ # Wait for instance to have public IP
while not instance.public_ip_address:
logger.warning("waiting for ip to be available")
sleep(5)
instance.reload()
- while True:
- sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
- if sock.connect_ex((instance.public_ip_address, 22)) == 0:
- break
- else:
- logger.warning("waiting for ssh to be available")
- sleep(10)
-
- def get_ssh_connection(instance_ip, ssh_identity_file, max_retries=10):
- for attempt in range(max_retries):
- try:
- return testinfra.get_host(
- f"paramiko://ubuntu@{instance_ip}?timeout=60",
- ssh_identity_file=ssh_identity_file,
- )
- except Exception as e:
- if attempt == max_retries - 1:
- raise
- logger.warning(
- f"Ssh connection failed, retrying: {attempt + 1}/{max_retries} failed, retrying ..."
- )
- sleep(5)
-
- host = get_ssh_connection(
- # paramiko is an ssh backend
+ # Create single SSH connection
+ ssh = get_ssh_connection(
instance.public_ip_address,
temp_key.get_priv_key_file(),
)
- def is_healthy(host, instance_ip, ssh_identity_file) -> bool:
+ # Check PostgreSQL data directory
+ logger.info("Checking PostgreSQL data directory...")
+ result = run_ssh_command(ssh, "ls -la /var/lib/postgresql")
+ if result['succeeded']:
+ logger.info("PostgreSQL data directory contents:\n" + result['stdout'])
+ else:
+ logger.warning("Failed to list PostgreSQL data directory: " + result['stderr'])
+
+ # Wait for init.sh to complete
+ logger.info("Waiting for init.sh to complete...")
+ max_attempts = 60 # 5 minutes
+ attempt = 0
+ while attempt < max_attempts:
+ try:
+ result = run_ssh_command(ssh, "test -f /var/lib/init-complete")
+ if result['succeeded']:
+ logger.info("init.sh has completed")
+ break
+ except Exception as e:
+ logger.warning(f"Error checking init.sh status: {str(e)}")
+
+ attempt += 1
+ logger.warning(f"Waiting for init.sh to complete (attempt {attempt}/{max_attempts})")
+ sleep(5)
+
+ if attempt >= max_attempts:
+ logger.error("init.sh failed to complete within the timeout period")
+ instance.terminate()
+ raise TimeoutError("init.sh failed to complete within the timeout period")
+
+ def is_healthy(ssh) -> bool:
health_checks = [
- (
- "postgres",
- lambda h: h.run("sudo -u postgres /usr/bin/pg_isready -U postgres"),
- ),
- (
- "adminapi",
- lambda h: h.run(
- f"curl -sf -k --connect-timeout 30 --max-time 60 https://localhost:8085/health -H 'apikey: {supabase_admin_key}'"
- ),
- ),
- (
- "postgrest",
- lambda h: h.run(
- "curl -sf --connect-timeout 30 --max-time 60 http://localhost:3001/ready"
- ),
- ),
- (
- "gotrue",
- lambda h: h.run(
- "curl -sf --connect-timeout 30 --max-time 60 http://localhost:8081/health"
- ),
- ),
- ("kong", lambda h: h.run("sudo kong health")),
- ("fail2ban", lambda h: h.run("sudo fail2ban-client status")),
+ ("postgres", "sudo -u postgres /usr/bin/pg_isready -U postgres"),
+ ("adminapi", f"curl -sf -k --connect-timeout 30 --max-time 60 https://localhost:8085/health -H 'apikey: {supabase_admin_key}'"),
+ ("postgrest", "curl -sf --connect-timeout 30 --max-time 60 http://localhost:3001/ready"),
+ ("gotrue", "curl -sf --connect-timeout 30 --max-time 60 http://localhost:8081/health"),
+ ("kong", "sudo kong health"),
+ ("fail2ban", "sudo fail2ban-client status"),
]
- for service, check in health_checks:
+ for service, command in health_checks:
try:
- cmd = check(host)
- if cmd.failed is True:
+ result = run_ssh_command(ssh, command)
+ if not result['succeeded']:
logger.warning(f"{service} not ready")
return False
except Exception:
- logger.warning(
- f"Connection failed during {service} check, attempting reconnect..."
- )
- host = get_ssh_connection(instance_ip, ssh_identity_file)
+ logger.warning(f"Connection failed during {service} check")
return False
return True
while True:
- if is_healthy(
- host=host,
- instance_ip=instance.public_ip_address,
- ssh_identity_file=temp_key.get_priv_key_file(),
- ):
+ if is_healthy(ssh):
break
sleep(1)
- # return a testinfra connection to the instance
- yield host
+ # Return both the SSH connection and instance IP for use in tests
+ yield {
+ 'ssh': ssh,
+ 'ip': instance.public_ip_address
+ }
# at the end of the test suite, destroy the instance
instance.terminate()
def test_postgrest_is_running(host):
- postgrest = host.service("postgrest")
- assert postgrest.is_running
+ """Check if postgrest service is running using our SSH connection."""
+ result = run_ssh_command(host['ssh'], "systemctl is-active postgrest")
+ assert result['succeeded'] and result['stdout'].strip() == 'active', "PostgREST service is not running"
def test_postgrest_responds_to_requests(host):
+ """Test if PostgREST responds to requests."""
res = requests.get(
- f"http://{host.backend.get_hostname()}/rest/v1/",
+ f"http://{host['ip']}/rest/v1/",
headers={
"apikey": anon_key,
"authorization": f"Bearer {anon_key}",
@@ -367,8 +401,9 @@ def test_postgrest_responds_to_requests(host):
def test_postgrest_can_connect_to_db(host):
+ """Test if PostgREST can connect to the database."""
res = requests.get(
- f"http://{host.backend.get_hostname()}/rest/v1/buckets",
+ f"http://{host['ip']}/rest/v1/buckets",
headers={
"apikey": service_role_key,
"authorization": f"Bearer {service_role_key}",
@@ -378,14 +413,10 @@ def test_postgrest_can_connect_to_db(host):
assert res.ok
-# There would be an error if the `apikey` query parameter isn't removed,
-# since PostgREST treats query parameters as conditions.
-#
-# Worth testing since remove_apikey_query_parameters uses regexp instead
-# of parsed query parameters.
def test_postgrest_starting_apikey_query_parameter_is_removed(host):
+ """Test if PostgREST removes apikey query parameter at start."""
res = requests.get(
- f"http://{host.backend.get_hostname()}/rest/v1/buckets",
+ f"http://{host['ip']}/rest/v1/buckets",
headers={
"accept-profile": "storage",
},
@@ -399,8 +430,9 @@ def test_postgrest_starting_apikey_query_parameter_is_removed(host):
def test_postgrest_middle_apikey_query_parameter_is_removed(host):
+ """Test if PostgREST removes apikey query parameter in middle."""
res = requests.get(
- f"http://{host.backend.get_hostname()}/rest/v1/buckets",
+ f"http://{host['ip']}/rest/v1/buckets",
headers={
"accept-profile": "storage",
},
@@ -414,8 +446,9 @@ def test_postgrest_middle_apikey_query_parameter_is_removed(host):
def test_postgrest_ending_apikey_query_parameter_is_removed(host):
+ """Test if PostgREST removes apikey query parameter at end."""
res = requests.get(
- f"http://{host.backend.get_hostname()}/rest/v1/buckets",
+ f"http://{host['ip']}/rest/v1/buckets",
headers={
"accept-profile": "storage",
},
@@ -428,14 +461,10 @@ def test_postgrest_ending_apikey_query_parameter_is_removed(host):
assert res.ok
-# There would be an error if the empty key query parameter isn't removed,
-# since PostgREST treats empty key query parameters as malformed input.
-#
-# Worth testing since remove_apikey_and_empty_key_query_parameters uses regexp instead
-# of parsed query parameters.
def test_postgrest_starting_empty_key_query_parameter_is_removed(host):
+ """Test if PostgREST removes empty key query parameter at start."""
res = requests.get(
- f"http://{host.backend.get_hostname()}/rest/v1/buckets",
+ f"http://{host['ip']}/rest/v1/buckets",
headers={
"accept-profile": "storage",
},
@@ -449,8 +478,9 @@ def test_postgrest_starting_empty_key_query_parameter_is_removed(host):
def test_postgrest_middle_empty_key_query_parameter_is_removed(host):
+ """Test if PostgREST removes empty key query parameter in middle."""
res = requests.get(
- f"http://{host.backend.get_hostname()}/rest/v1/buckets",
+ f"http://{host['ip']}/rest/v1/buckets",
headers={
"accept-profile": "storage",
},
@@ -464,8 +494,9 @@ def test_postgrest_middle_empty_key_query_parameter_is_removed(host):
def test_postgrest_ending_empty_key_query_parameter_is_removed(host):
+ """Test if PostgREST removes empty key query parameter at end."""
res = requests.get(
- f"http://{host.backend.get_hostname()}/rest/v1/buckets",
+ f"http://{host['ip']}/rest/v1/buckets",
headers={
"accept-profile": "storage",
},
[parent: 4c10c24c52df]