agora inbox for pgsql-hackers@postgresql.orghelp / color / mirror / Atom feed
[PATCH v20 4/6] Move conversion of a "historic" to MVCC snapshot to a separate function. 249+ messages / 2 participants [nested] [flat]
* [PATCH v20 4/6] Move conversion of a "historic" to MVCC snapshot to a separate function. @ 2025-08-11 13:23 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 249+ messages in thread From: Antonin Houska @ 2025-08-11 13:23 UTC (permalink / raw) The conversion is now handled by SnapBuildMVCCFromHistoric(). REPACK CONCURRENTLY will also need it. --- src/backend/replication/logical/snapbuild.c | 51 +++++++++++++++++---- src/backend/utils/time/snapmgr.c | 3 +- src/include/replication/snapbuild.h | 1 + src/include/utils/snapmgr.h | 1 + 4 files changed, 45 insertions(+), 11 deletions(-) diff --git a/src/backend/replication/logical/snapbuild.c b/src/backend/replication/logical/snapbuild.c index 98ddee20929..a2f1803622c 100644 --- a/src/backend/replication/logical/snapbuild.c +++ b/src/backend/replication/logical/snapbuild.c @@ -440,10 +440,7 @@ Snapshot SnapBuildInitialSnapshot(SnapBuild *builder) { Snapshot snap; - TransactionId xid; TransactionId safeXid; - TransactionId *newxip; - int newxcnt = 0; Assert(XactIsoLevel == XACT_REPEATABLE_READ); Assert(builder->building_full_snapshot); @@ -485,6 +482,31 @@ SnapBuildInitialSnapshot(SnapBuild *builder) MyProc->xmin = snap->xmin; + /* Convert the historic snapshot to MVCC snapshot. */ + return SnapBuildMVCCFromHistoric(snap, true); +} + +/* + * Turn a historic MVCC snapshot into an ordinary MVCC snapshot. + * + * Unlike a regular (non-historic) MVCC snapshot, the xip array of this + * snapshot contains not only running main transactions, but also their + * subtransactions. This difference does has no impact on XidInMVCCSnapshot(). + * + * Pass true for 'in_place' if you don't care about modifying the source + * snapshot. If you need a new instance, and one that was allocated as a + * single chunk of memory, pass false. + */ +Snapshot +SnapBuildMVCCFromHistoric(Snapshot snapshot, bool in_place) +{ + TransactionId xid; + TransactionId *oldxip = snapshot->xip; + uint32 oldxcnt = snapshot->xcnt; + TransactionId *newxip; + int newxcnt = 0; + Snapshot result; + /* allocate in transaction context */ newxip = (TransactionId *) palloc(sizeof(TransactionId) * GetMaxSnapshotXidCount()); @@ -495,7 +517,7 @@ SnapBuildInitialSnapshot(SnapBuild *builder) * classical snapshot by marking all non-committed transactions as * in-progress. This can be expensive. */ - for (xid = snap->xmin; NormalTransactionIdPrecedes(xid, snap->xmax);) + for (xid = snapshot->xmin; NormalTransactionIdPrecedes(xid, snapshot->xmax);) { void *test; @@ -503,7 +525,7 @@ SnapBuildInitialSnapshot(SnapBuild *builder) * Check whether transaction committed using the decoding snapshot * meaning of ->xip. */ - test = bsearch(&xid, snap->xip, snap->xcnt, + test = bsearch(&xid, snapshot->xip, snapshot->xcnt, sizeof(TransactionId), xidComparator); if (test == NULL) @@ -520,11 +542,22 @@ SnapBuildInitialSnapshot(SnapBuild *builder) } /* adjust remaining snapshot fields as needed */ - snap->snapshot_type = SNAPSHOT_MVCC; - snap->xcnt = newxcnt; - snap->xip = newxip; + snapshot->xcnt = newxcnt; + snapshot->xip = newxip; - return snap; + if (in_place) + result = snapshot; + else + { + result = CopySnapshot(snapshot); + + /* Restore the original values so the source is intact. */ + snapshot->xip = oldxip; + snapshot->xcnt = oldxcnt; + } + result->snapshot_type = SNAPSHOT_MVCC; + + return result; } /* diff --git a/src/backend/utils/time/snapmgr.c b/src/backend/utils/time/snapmgr.c index 65561cc6bc3..bc7840052fe 100644 --- a/src/backend/utils/time/snapmgr.c +++ b/src/backend/utils/time/snapmgr.c @@ -212,7 +212,6 @@ typedef struct ExportedSnapshot static List *exportedSnapshots = NIL; /* Prototypes for local functions */ -static Snapshot CopySnapshot(Snapshot snapshot); static void UnregisterSnapshotNoOwner(Snapshot snapshot); static void FreeSnapshot(Snapshot snapshot); static void SnapshotResetXmin(void); @@ -602,7 +601,7 @@ SetTransactionSnapshot(Snapshot sourcesnap, VirtualTransactionId *sourcevxid, * The copy is palloc'd in TopTransactionContext and has initial refcounts set * to 0. The returned snapshot has the copied flag set. */ -static Snapshot +Snapshot CopySnapshot(Snapshot snapshot) { Snapshot newsnap; diff --git a/src/include/replication/snapbuild.h b/src/include/replication/snapbuild.h index 44031dcf6e3..6d4d2d1814c 100644 --- a/src/include/replication/snapbuild.h +++ b/src/include/replication/snapbuild.h @@ -73,6 +73,7 @@ extern void FreeSnapshotBuilder(SnapBuild *builder); extern void SnapBuildSnapDecRefcount(Snapshot snap); extern Snapshot SnapBuildInitialSnapshot(SnapBuild *builder); +extern Snapshot SnapBuildMVCCFromHistoric(Snapshot snapshot, bool in_place); extern const char *SnapBuildExportSnapshot(SnapBuild *builder); extern void SnapBuildClearExportedSnapshot(void); extern void SnapBuildResetExportedSnapshotState(void); diff --git a/src/include/utils/snapmgr.h b/src/include/utils/snapmgr.h index 604c1f90216..f65f83c85cd 100644 --- a/src/include/utils/snapmgr.h +++ b/src/include/utils/snapmgr.h @@ -63,6 +63,7 @@ extern Snapshot GetTransactionSnapshot(void); extern Snapshot GetLatestSnapshot(void); extern void SnapshotSetCommandId(CommandId curcid); +extern Snapshot CopySnapshot(Snapshot snapshot); extern Snapshot GetCatalogSnapshot(Oid relid); extern Snapshot GetNonHistoricCatalogSnapshot(Oid relid); extern void InvalidateCatalogSnapshot(void); -- 2.39.5 --3q6txozlgl6t37td Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v20-0005-Add-CONCURRENTLY-option-to-REPACK-command.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment @ 2026-06-10 18:49 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 249+ messages in thread From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw) Previously we hardcoded the amount of concurrency in the CI definition. This practice IIRC originated from a) using make, where the concurrency is not sourced from the current system b) early cirrus-ci macos runners, which slowed down a lot with full concurrency. Hardcoding the concurrency is problematic as e.g. private repositories have lower concurrency and larger github runners have more cores. For most of the jobs we can rely on just using meson's and ninja's logic for getting the current core count. For the autoconf we can use the nproc helper. To make this at least somewhat understandable, I added the existing sysinfo step to show the number of cores (and made it a bit easier to understand the information already printed). Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33 --- .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index ab7950f6f14..9384ec0d902 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -40,14 +40,6 @@ env: # concurrent jobs and retrying older runs have a chance of working. CLONE_DEPTH: 500 - # At the moment all jobs use 4vcore runners, and none seems to benefit from - # increasing concurrency further. - BUILD_JOBS: 4 - - # It's possible that some jobs benefit from an increased test concurrency, - # but a default of 4 is a safe bet. Individual jobs can override. - TEST_JOBS: 4 - CCACHE_MAXSIZE: "250M" CCACHE_DIR: ${{ github.workspace }}/ccache_dir @@ -215,10 +207,19 @@ jobs: - &nix_sysinfo_step name: sysinfo run: | - id - uname -a + echo -n "Number of cores: " + nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu + + echo "id: $(id)" + echo "uname: $(uname -a)" + + echo ::group::ulimit ulimit -a -H && ulimit -a -S + echo ::endgroup:: + + echo ::group::env env + echo ::endgroup:: - name: Parse ci-os-only id: os @@ -352,7 +353,7 @@ jobs: - name: Build shell: *su_postgres_shell run: &ninja_build_cmd | - ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build ${{env.MBUILD_TARGET}} ninja -C build -t missingdeps # Decide if it's worth uploading a new version of the ccache cache. If @@ -405,7 +406,7 @@ jobs: meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1 echo ::endgroup:: - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}} + meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}} - &linux_collect_cores_step name: Core backtraces @@ -511,7 +512,7 @@ jobs: - name: Build shell: *su_postgres_shell run: | - make -s -j${BUILD_JOBS} world-bin + make -s -j$(nproc) world-bin - *ccache_decide_save_step - *ccache_save_step @@ -519,7 +520,7 @@ jobs: - name: Test world shell: *su_postgres_shell run: | - make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} + make -s ${CHECK} ${CHECKFLAGS} -j$(nproc) - *linux_collect_cores_step - *upload_logs_step @@ -611,7 +612,7 @@ jobs: build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start # Run the tests supporting running against an already running - meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running + meson test ${{env.MTEST_ARGS}} --setup running build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop @@ -893,9 +894,19 @@ jobs: - name: Sysinfo run: | + echo Number of cores: + bash -c nproc + + echo codepage: chcp + + echo ::group::systeminfo systeminfo + echo ::endgroup:: + + echo ::group::env set + echo ::endgroup:: # The TAP tests build an initdb template under build/tmp_install and # then `robocopy` it into per-test data directories. Robocopy with the @@ -1198,7 +1209,7 @@ jobs: meson setup -Dwerror=true $CONF build-meson echo "::endgroup::" - ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}} + ninja -C build-meson --quiet ${{env.MBUILD_TARGET}} # gcc, cassert on, dtrace off - name: gcc warnings + (cassert) @@ -1266,8 +1277,8 @@ jobs: CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" echo "::endgroup::" - make -s -j${{env.BUILD_JOBS}} clean - make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \ + make -s -j$(nproc) clean + make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \ headerscheck cpluspluscheck \ EXTRAFLAGS='-fmax-errors=10' -- 2.54.0.450.g9ac3f193c0 --mr2uqtieh2e2xvha Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13a-0008-ci-Make-runs-on-overridable.patch" ^ permalink raw reply [nested|flat] 249+ messages in thread
end of thread, other threads:[~2026-06-10 18:49 UTC | newest] Thread overview: 249+ messages (download: mbox mbox.gz follow: Atom feed) -- links below jump to the message on this page -- 2025-08-11 13:23 [PATCH v20 4/6] Move conversion of a "historic" to MVCC snapshot to a separate function. Antonin Houska <ah@cybertec.at> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de> 2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
This inbox is served by agora; see mirroring instructions for how to clone and mirror all data and code used for this inbox