agora inbox for [email protected]
help / color / mirror / Atom feed[PATCH v53 3/7] Rename cluster.c/h -> repack.c/h
152+ messages / 2 participants
[nested] [flat]
* [PATCH v53 3/7] Rename cluster.c/h -> repack.c/h
@ 2026-03-31 16:55 Álvaro Herrera <[email protected]>
0 siblings, 0 replies; 152+ messages in thread
From: Álvaro Herrera @ 2026-03-31 16:55 UTC (permalink / raw)
---
src/backend/commands/Makefile | 2 +-
src/backend/commands/matview.c | 2 +-
src/backend/commands/meson.build | 2 +-
src/backend/commands/{cluster.c => repack.c} | 6 +++---
src/backend/commands/tablecmds.c | 2 +-
src/backend/commands/vacuum.c | 6 +++---
src/backend/storage/ipc/procsignal.c | 1 +
src/backend/tcop/postgres.c | 1 +
src/backend/tcop/utility.c | 2 +-
src/include/commands/{cluster.h => repack.h} | 12 ++++++------
10 files changed, 19 insertions(+), 17 deletions(-)
rename src/backend/commands/{cluster.c => repack.c} (99%)
rename src/include/commands/{cluster.h => repack.h} (90%)
diff --git a/src/backend/commands/Makefile b/src/backend/commands/Makefile
index c10fdba2bbb..fe1bba3a9b9 100644
--- a/src/backend/commands/Makefile
+++ b/src/backend/commands/Makefile
@@ -18,7 +18,6 @@ OBJS = \
amcmds.o \
analyze.o \
async.o \
- cluster.o \
collationcmds.o \
comment.o \
constraint.o \
@@ -51,6 +50,7 @@ OBJS = \
proclang.o \
propgraphcmds.o \
publicationcmds.o \
+ repack.o \
schemacmds.o \
seclabel.o \
sequence.o \
diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c
index d3be8939011..5db4fe75dce 100644
--- a/src/backend/commands/matview.c
+++ b/src/backend/commands/matview.c
@@ -24,8 +24,8 @@
#include "catalog/namespace.h"
#include "catalog/pg_am.h"
#include "catalog/pg_opclass.h"
-#include "commands/cluster.h"
#include "commands/matview.h"
+#include "commands/repack.h"
#include "commands/tablecmds.h"
#include "commands/tablespace.h"
#include "executor/executor.h"
diff --git a/src/backend/commands/meson.build b/src/backend/commands/meson.build
index 90c7e37a429..f624aae74af 100644
--- a/src/backend/commands/meson.build
+++ b/src/backend/commands/meson.build
@@ -6,7 +6,6 @@ backend_sources += files(
'amcmds.c',
'analyze.c',
'async.c',
- 'cluster.c',
'collationcmds.c',
'comment.c',
'constraint.c',
@@ -39,6 +38,7 @@ backend_sources += files(
'proclang.c',
'propgraphcmds.c',
'publicationcmds.c',
+ 'repack.c',
'schemacmds.c',
'seclabel.c',
'sequence.c',
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/repack.c
similarity index 99%
rename from src/backend/commands/cluster.c
rename to src/backend/commands/repack.c
index f241e18b153..20f0a572236 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/repack.c
@@ -1,6 +1,6 @@
/*-------------------------------------------------------------------------
*
- * cluster.c
+ * repack.c
* REPACK a table; formerly known as CLUSTER. VACUUM FULL also uses
* parts of this code.
*
@@ -10,7 +10,7 @@
*
*
* IDENTIFICATION
- * src/backend/commands/cluster.c
+ * src/backend/commands/repack.c
*
*-------------------------------------------------------------------------
*/
@@ -33,9 +33,9 @@
#include "catalog/pg_am.h"
#include "catalog/pg_inherits.h"
#include "catalog/toasting.h"
-#include "commands/cluster.h"
#include "commands/defrem.h"
#include "commands/progress.h"
+#include "commands/repack.h"
#include "commands/tablecmds.h"
#include "commands/vacuum.h"
#include "miscadmin.h"
diff --git a/src/backend/commands/tablecmds.c b/src/backend/commands/tablecmds.c
index 0ce2e81f9c2..e2882a50b3b 100644
--- a/src/backend/commands/tablecmds.c
+++ b/src/backend/commands/tablecmds.c
@@ -57,10 +57,10 @@
#include "catalog/storage.h"
#include "catalog/storage_xlog.h"
#include "catalog/toasting.h"
-#include "commands/cluster.h"
#include "commands/comment.h"
#include "commands/defrem.h"
#include "commands/event_trigger.h"
+#include "commands/repack.h"
#include "commands/sequence.h"
#include "commands/tablecmds.h"
#include "commands/tablespace.h"
diff --git a/src/backend/commands/vacuum.c b/src/backend/commands/vacuum.c
index 0ed363d1c85..b179b62b5c8 100644
--- a/src/backend/commands/vacuum.c
+++ b/src/backend/commands/vacuum.c
@@ -9,7 +9,7 @@
*
* VACUUM for heap AM is implemented in vacuumlazy.c, parallel vacuum in
* vacuumparallel.c, ANALYZE in analyze.c, and VACUUM FULL is a variant of
- * CLUSTER, handled in cluster.c.
+ * REPACK, handled in repack.c.
*
*
* Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group
@@ -38,9 +38,9 @@
#include "catalog/pg_database.h"
#include "catalog/pg_inherits.h"
#include "commands/async.h"
-#include "commands/cluster.h"
#include "commands/defrem.h"
#include "commands/progress.h"
+#include "commands/repack.h"
#include "commands/vacuum.h"
#include "miscadmin.h"
#include "nodes/makefuncs.h"
@@ -2293,7 +2293,7 @@ vacuum_rel(Oid relid, RangeVar *relation, VacuumParams params,
if ((params.options & VACOPT_VERBOSE) != 0)
cluster_params.options |= CLUOPT_VERBOSE;
- /* VACUUM FULL is a variant of REPACK; see cluster.c */
+ /* VACUUM FULL is a variant of REPACK; see repack.c */
cluster_rel(REPACK_COMMAND_VACUUMFULL, rel, InvalidOid,
&cluster_params);
/* cluster_rel closes the relation, but keeps lock */
diff --git a/src/backend/storage/ipc/procsignal.c b/src/backend/storage/ipc/procsignal.c
index f1ab3aa3fe0..02d28df1c6a 100644
--- a/src/backend/storage/ipc/procsignal.c
+++ b/src/backend/storage/ipc/procsignal.c
@@ -19,6 +19,7 @@
#include "access/parallel.h"
#include "commands/async.h"
+#include "commands/repack.h"
#include "miscadmin.h"
#include "pgstat.h"
#include "port/pg_bitutils.h"
diff --git a/src/backend/tcop/postgres.c b/src/backend/tcop/postgres.c
index 10be60011ad..9fbaa5c00f0 100644
--- a/src/backend/tcop/postgres.c
+++ b/src/backend/tcop/postgres.c
@@ -39,6 +39,7 @@
#include "commands/event_trigger.h"
#include "commands/explain_state.h"
#include "commands/prepare.h"
+#include "commands/repack.h"
#include "common/pg_prng.h"
#include "jit/jit.h"
#include "libpq/libpq.h"
diff --git a/src/backend/tcop/utility.c b/src/backend/tcop/utility.c
index 2b609bfc824..5f8c766c4be 100644
--- a/src/backend/tcop/utility.c
+++ b/src/backend/tcop/utility.c
@@ -26,7 +26,6 @@
#include "catalog/toasting.h"
#include "commands/alter.h"
#include "commands/async.h"
-#include "commands/cluster.h"
#include "commands/collationcmds.h"
#include "commands/comment.h"
#include "commands/conversioncmds.h"
@@ -46,6 +45,7 @@
#include "commands/proclang.h"
#include "commands/propgraphcmds.h"
#include "commands/publicationcmds.h"
+#include "commands/repack.h"
#include "commands/schemacmds.h"
#include "commands/seclabel.h"
#include "commands/sequence.h"
diff --git a/src/include/commands/cluster.h b/src/include/commands/repack.h
similarity index 90%
rename from src/include/commands/cluster.h
rename to src/include/commands/repack.h
index d6b62c747e8..85061158b0c 100644
--- a/src/include/commands/cluster.h
+++ b/src/include/commands/repack.h
@@ -1,17 +1,17 @@
/*-------------------------------------------------------------------------
*
- * cluster.h
- * header file for postgres cluster command stuff
+ * repack.h
+ * header file for the REPACK command
*
* Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group
* Portions Copyright (c) 1994-5, Regents of the University of California
*
- * src/include/commands/cluster.h
+ * src/include/commands/repack.h
*
*-------------------------------------------------------------------------
*/
-#ifndef CLUSTER_H
-#define CLUSTER_H
+#ifndef REPACK_H
+#define REPACK_H
#include "nodes/parsenodes.h"
#include "parser/parse_node.h"
@@ -52,4 +52,4 @@ extern void finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap,
MultiXactId cutoffMulti,
char newrelpersistence);
-#endif /* CLUSTER_H */
+#endif /* REPACK_H */
--
2.47.3
--qr3jlalmmcpkiodg
Content-Type: text/x-diff; charset=utf-8
Content-Disposition: attachment;
filename="v53-0004-Add-CONCURRENTLY-option-to-REPACK-command.patch"
^ permalink raw reply [nested|flat] 152+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
0 siblings, 0 replies; 152+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 152+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
0 siblings, 0 replies; 152+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 152+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
0 siblings, 0 replies; 152+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 152+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
0 siblings, 0 replies; 152+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 152+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
0 siblings, 0 replies; 152+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 152+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
0 siblings, 0 replies; 152+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 152+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
0 siblings, 0 replies; 152+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 152+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
0 siblings, 0 replies; 152+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 152+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
0 siblings, 0 replies; 152+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 152+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
0 siblings, 0 replies; 152+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 152+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
0 siblings, 0 replies; 152+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 152+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
0 siblings, 0 replies; 152+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 152+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
0 siblings, 0 replies; 152+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 152+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
0 siblings, 0 replies; 152+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 152+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
0 siblings, 0 replies; 152+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 152+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
0 siblings, 0 replies; 152+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 152+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
0 siblings, 0 replies; 152+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 152+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
0 siblings, 0 replies; 152+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 152+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
0 siblings, 0 replies; 152+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 152+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
0 siblings, 0 replies; 152+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 152+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
0 siblings, 0 replies; 152+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 152+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
0 siblings, 0 replies; 152+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 152+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
0 siblings, 0 replies; 152+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 152+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
0 siblings, 0 replies; 152+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 152+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
0 siblings, 0 replies; 152+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 152+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
0 siblings, 0 replies; 152+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 152+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
0 siblings, 0 replies; 152+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 152+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
0 siblings, 0 replies; 152+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 152+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
0 siblings, 0 replies; 152+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 152+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
0 siblings, 0 replies; 152+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 152+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
0 siblings, 0 replies; 152+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 152+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
0 siblings, 0 replies; 152+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 152+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
0 siblings, 0 replies; 152+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 152+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
0 siblings, 0 replies; 152+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 152+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
0 siblings, 0 replies; 152+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 152+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
0 siblings, 0 replies; 152+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 152+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
0 siblings, 0 replies; 152+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 152+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
0 siblings, 0 replies; 152+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 152+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
0 siblings, 0 replies; 152+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 152+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
0 siblings, 0 replies; 152+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 152+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
0 siblings, 0 replies; 152+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 152+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
0 siblings, 0 replies; 152+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 152+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
0 siblings, 0 replies; 152+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 152+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
0 siblings, 0 replies; 152+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 152+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
0 siblings, 0 replies; 152+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 152+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
0 siblings, 0 replies; 152+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 152+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
0 siblings, 0 replies; 152+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 152+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
0 siblings, 0 replies; 152+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 152+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
0 siblings, 0 replies; 152+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 152+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
0 siblings, 0 replies; 152+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 152+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
0 siblings, 0 replies; 152+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 152+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
0 siblings, 0 replies; 152+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 152+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
0 siblings, 0 replies; 152+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 152+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
0 siblings, 0 replies; 152+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 152+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
0 siblings, 0 replies; 152+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 152+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
0 siblings, 0 replies; 152+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 152+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
0 siblings, 0 replies; 152+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 152+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
0 siblings, 0 replies; 152+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 152+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
0 siblings, 0 replies; 152+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 152+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
0 siblings, 0 replies; 152+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 152+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
0 siblings, 0 replies; 152+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 152+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
0 siblings, 0 replies; 152+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 152+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
0 siblings, 0 replies; 152+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 152+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
0 siblings, 0 replies; 152+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 152+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
0 siblings, 0 replies; 152+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 152+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
0 siblings, 0 replies; 152+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 152+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
0 siblings, 0 replies; 152+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 152+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
0 siblings, 0 replies; 152+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 152+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
0 siblings, 0 replies; 152+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 152+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
0 siblings, 0 replies; 152+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 152+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
0 siblings, 0 replies; 152+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 152+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
0 siblings, 0 replies; 152+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 152+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
0 siblings, 0 replies; 152+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 152+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
0 siblings, 0 replies; 152+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 152+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
0 siblings, 0 replies; 152+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 152+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
0 siblings, 0 replies; 152+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 152+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
0 siblings, 0 replies; 152+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 152+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
0 siblings, 0 replies; 152+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 152+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
0 siblings, 0 replies; 152+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 152+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
0 siblings, 0 replies; 152+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 152+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
0 siblings, 0 replies; 152+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 152+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
0 siblings, 0 replies; 152+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 152+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
0 siblings, 0 replies; 152+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 152+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
0 siblings, 0 replies; 152+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 152+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
0 siblings, 0 replies; 152+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 152+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
0 siblings, 0 replies; 152+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 152+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
0 siblings, 0 replies; 152+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 152+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
0 siblings, 0 replies; 152+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 152+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
0 siblings, 0 replies; 152+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 152+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
0 siblings, 0 replies; 152+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 152+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
0 siblings, 0 replies; 152+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 152+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
0 siblings, 0 replies; 152+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 152+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
0 siblings, 0 replies; 152+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 152+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
0 siblings, 0 replies; 152+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 152+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
0 siblings, 0 replies; 152+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 152+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
0 siblings, 0 replies; 152+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 152+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
0 siblings, 0 replies; 152+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 152+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
0 siblings, 0 replies; 152+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 152+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
0 siblings, 0 replies; 152+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 152+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
0 siblings, 0 replies; 152+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 152+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
0 siblings, 0 replies; 152+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 152+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
0 siblings, 0 replies; 152+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 152+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
0 siblings, 0 replies; 152+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 152+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
0 siblings, 0 replies; 152+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 152+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
0 siblings, 0 replies; 152+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 152+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
0 siblings, 0 replies; 152+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 152+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
0 siblings, 0 replies; 152+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 152+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
0 siblings, 0 replies; 152+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 152+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
0 siblings, 0 replies; 152+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 152+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
0 siblings, 0 replies; 152+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 152+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
0 siblings, 0 replies; 152+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 152+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
0 siblings, 0 replies; 152+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 152+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
0 siblings, 0 replies; 152+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 152+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
0 siblings, 0 replies; 152+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 152+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
0 siblings, 0 replies; 152+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 152+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
0 siblings, 0 replies; 152+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 152+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
0 siblings, 0 replies; 152+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 152+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
0 siblings, 0 replies; 152+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 152+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
0 siblings, 0 replies; 152+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 152+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
0 siblings, 0 replies; 152+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 152+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
0 siblings, 0 replies; 152+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 152+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
0 siblings, 0 replies; 152+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 152+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
0 siblings, 0 replies; 152+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 152+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
0 siblings, 0 replies; 152+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 152+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
0 siblings, 0 replies; 152+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 152+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
0 siblings, 0 replies; 152+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 152+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
0 siblings, 0 replies; 152+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 152+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
0 siblings, 0 replies; 152+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 152+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
0 siblings, 0 replies; 152+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 152+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
0 siblings, 0 replies; 152+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 152+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
0 siblings, 0 replies; 152+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 152+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
0 siblings, 0 replies; 152+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 152+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
0 siblings, 0 replies; 152+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 152+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
0 siblings, 0 replies; 152+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 152+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
0 siblings, 0 replies; 152+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 152+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
0 siblings, 0 replies; 152+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 152+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
0 siblings, 0 replies; 152+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 152+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
0 siblings, 0 replies; 152+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 152+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
0 siblings, 0 replies; 152+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 152+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
0 siblings, 0 replies; 152+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 152+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
0 siblings, 0 replies; 152+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 152+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
0 siblings, 0 replies; 152+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 152+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
0 siblings, 0 replies; 152+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 152+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
0 siblings, 0 replies; 152+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 152+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
0 siblings, 0 replies; 152+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 152+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
0 siblings, 0 replies; 152+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 152+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
0 siblings, 0 replies; 152+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 152+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
0 siblings, 0 replies; 152+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 152+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
0 siblings, 0 replies; 152+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 152+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
0 siblings, 0 replies; 152+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 152+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
0 siblings, 0 replies; 152+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 152+ messages in thread
end of thread, other threads:[~2026-06-10 18:49 UTC | newest]
Thread overview: 152+ messages (download: mbox mbox.gz follow: Atom feed)
-- links below jump to the message on this page --
2026-03-31 16:55 [PATCH v53 3/7] Rename cluster.c/h -> repack.c/h Álvaro Herrera <[email protected]>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <[email protected]>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <[email protected]>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <[email protected]>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <[email protected]>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <[email protected]>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <[email protected]>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <[email protected]>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <[email protected]>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <[email protected]>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <[email protected]>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <[email protected]>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <[email protected]>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <[email protected]>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <[email protected]>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <[email protected]>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <[email protected]>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <[email protected]>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <[email protected]>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <[email protected]>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <[email protected]>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <[email protected]>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <[email protected]>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <[email protected]>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <[email protected]>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <[email protected]>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <[email protected]>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <[email protected]>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <[email protected]>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <[email protected]>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <[email protected]>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <[email protected]>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <[email protected]>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <[email protected]>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <[email protected]>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <[email protected]>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <[email protected]>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <[email protected]>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <[email protected]>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <[email protected]>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <[email protected]>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <[email protected]>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <[email protected]>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <[email protected]>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <[email protected]>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <[email protected]>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <[email protected]>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <[email protected]>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <[email protected]>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <[email protected]>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <[email protected]>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <[email protected]>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <[email protected]>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <[email protected]>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <[email protected]>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <[email protected]>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <[email protected]>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <[email protected]>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <[email protected]>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <[email protected]>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <[email protected]>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <[email protected]>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <[email protected]>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <[email protected]>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <[email protected]>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <[email protected]>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <[email protected]>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <[email protected]>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <[email protected]>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <[email protected]>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <[email protected]>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <[email protected]>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <[email protected]>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <[email protected]>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <[email protected]>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <[email protected]>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <[email protected]>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <[email protected]>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <[email protected]>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <[email protected]>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <[email protected]>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <[email protected]>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <[email protected]>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <[email protected]>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <[email protected]>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <[email protected]>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <[email protected]>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <[email protected]>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <[email protected]>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <[email protected]>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <[email protected]>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <[email protected]>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <[email protected]>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <[email protected]>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <[email protected]>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <[email protected]>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <[email protected]>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <[email protected]>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <[email protected]>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <[email protected]>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <[email protected]>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <[email protected]>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <[email protected]>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <[email protected]>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <[email protected]>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <[email protected]>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <[email protected]>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <[email protected]>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <[email protected]>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <[email protected]>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <[email protected]>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <[email protected]>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <[email protected]>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <[email protected]>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <[email protected]>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <[email protected]>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <[email protected]>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <[email protected]>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <[email protected]>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <[email protected]>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <[email protected]>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <[email protected]>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <[email protected]>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <[email protected]>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <[email protected]>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <[email protected]>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <[email protected]>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <[email protected]>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <[email protected]>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <[email protected]>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <[email protected]>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <[email protected]>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <[email protected]>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <[email protected]>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <[email protected]>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <[email protected]>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <[email protected]>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <[email protected]>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <[email protected]>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <[email protected]>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <[email protected]>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <[email protected]>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <[email protected]>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <[email protected]>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <[email protected]>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <[email protected]>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <[email protected]>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <[email protected]>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <[email protected]>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <[email protected]>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <[email protected]>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <[email protected]>
This inbox is served by agora; see mirroring instructions
for how to clone and mirror all data and code used for this inbox