agora inbox for pgsql-hackers@postgresql.org  

help / color / mirror / Atom feed

[PATCH v33 03/11] Allow to prolong life span of transition tables until transaction end
249+ messages / 2 participants
[nested] [flat]

* [PATCH v33 03/11] Allow to prolong life span of transition tables until transaction end
@ 2019-12-20 01:09 Yugo Nagata <nagata@sraoss.co.jp>
  0 siblings, 0 replies; 249+ messages in thread

From: Yugo Nagata @ 2019-12-20 01:09 UTC (permalink / raw)

Originally, tuplestores of AFTER trigger's transition tables were
freed for each query depth. For our IVM implementation, we would like
to prolong life of the tuplestores because we have to preserve them
for a whole query assuming that some base tables might be changed
in some trigger functions.
---
 src/backend/commands/trigger.c | 83 ++++++++++++++++++++++++++++++++--
 src/include/commands/trigger.h |  2 +
 2 files changed, 81 insertions(+), 4 deletions(-)

diff --git a/src/backend/commands/trigger.c b/src/backend/commands/trigger.c
index 58b7fc5bbd..bd4a2219a4 100644
--- a/src/backend/commands/trigger.c
+++ b/src/backend/commands/trigger.c
@@ -3753,6 +3753,10 @@ typedef struct AfterTriggerEventList
  * end of the list, so it is relatively easy to discard them.  The event
  * list chunks themselves are stored in event_cxt.
  *
+ * prolonged_tuplestored is a list of transition table tuplestores whose
+ * life are prolonged to the end of the outmost query instead of each nested
+ * query.
+ *
  * query_depth is the current depth of nested AfterTriggerBeginQuery calls
  * (-1 when the stack is empty).
  *
@@ -3818,6 +3822,7 @@ typedef struct AfterTriggersData
 	SetConstraintState state;	/* the active S C state */
 	AfterTriggerEventList events;	/* deferred-event list */
 	MemoryContext event_cxt;	/* memory context for events, if any */
+	List   *prolonged_tuplestores;	/* list of prolonged tuplestores */
 
 	/* per-query-level data: */
 	AfterTriggersQueryData *query_stack;	/* array of structs shown below */
@@ -3853,6 +3858,7 @@ struct AfterTriggersTableData
 	bool		closed;			/* true when no longer OK to add tuples */
 	bool		before_trig_done;	/* did we already queue BS triggers? */
 	bool		after_trig_done;	/* did we already queue AS triggers? */
+	bool		prolonged;			/* are transition tables prolonged? */
 	AfterTriggerEventList after_trig_events;	/* if so, saved list pointer */
 
 	/*
@@ -3902,6 +3908,7 @@ static void TransitionTableAddTuple(EState *estate,
 									TupleTableSlot *original_insert_tuple,
 									Tuplestorestate *tuplestore);
 static void AfterTriggerFreeQuery(AfterTriggersQueryData *qs);
+static void release_or_prolong_tuplestore(Tuplestorestate *ts, bool prolonged);
 static SetConstraintState SetConstraintStateCreate(int numalloc);
 static SetConstraintState SetConstraintStateCopy(SetConstraintState origstate);
 static SetConstraintState SetConstraintStateAddItem(SetConstraintState state,
@@ -4781,6 +4788,45 @@ afterTriggerInvokeEvents(AfterTriggerEventList *events,
 }
 
 
+/*
+ * SetTransitionTablePreserved
+ *
+ * Prolong lifespan of transition tables corresponding specified relid and
+ * command type to the end of the outmost query instead of each nested query.
+ * This enables to use nested AFTER trigger's transition tables from outer
+ * query's triggers.  Currently, only immediate incremental view maintenance
+ * uses this.
+ */
+void
+SetTransitionTablePreserved(Oid relid, CmdType cmdType)
+{
+	AfterTriggersTableData *table;
+	AfterTriggersQueryData *qs;
+	bool		found = false;
+	ListCell   *lc;
+
+	/* Check state, like AfterTriggerSaveEvent. */
+	if (afterTriggers.query_depth < 0)
+		elog(ERROR, "SetTransitionTablePreserved() called outside of query");
+
+	qs = &afterTriggers.query_stack[afterTriggers.query_depth];
+
+	foreach(lc, qs->tables)
+	{
+		table = (AfterTriggersTableData *) lfirst(lc);
+		if (table->relid == relid && table->cmdType == cmdType &&
+			table->closed)
+		{
+			table->prolonged = true;
+			found = true;
+		}
+	}
+
+	if (!found)
+		elog(ERROR,"could not find table with OID %d and command type %d", relid, cmdType);
+}
+
+
 /*
  * GetAfterTriggersTableData
  *
@@ -4991,6 +5037,7 @@ AfterTriggerBeginXact(void)
 	 */
 	afterTriggers.firing_counter = (CommandId) 1;	/* mustn't be 0 */
 	afterTriggers.query_depth = -1;
+	afterTriggers.prolonged_tuplestores = NIL;
 
 	/*
 	 * Verify that there is no leftover state remaining.  If these assertions
@@ -5151,19 +5198,19 @@ AfterTriggerFreeQuery(AfterTriggersQueryData *qs)
 		ts = table->old_upd_tuplestore;
 		table->old_upd_tuplestore = NULL;
 		if (ts)
-			tuplestore_end(ts);
+			release_or_prolong_tuplestore(ts, table->prolonged);
 		ts = table->new_upd_tuplestore;
 		table->new_upd_tuplestore = NULL;
 		if (ts)
-			tuplestore_end(ts);
+			release_or_prolong_tuplestore(ts, table->prolonged);
 		ts = table->old_del_tuplestore;
 		table->old_del_tuplestore = NULL;
 		if (ts)
-			tuplestore_end(ts);
+			release_or_prolong_tuplestore(ts, table->prolonged);
 		ts = table->new_ins_tuplestore;
 		table->new_ins_tuplestore = NULL;
 		if (ts)
-			tuplestore_end(ts);
+			release_or_prolong_tuplestore(ts, table->prolonged);
 		if (table->storeslot)
 		{
 			TupleTableSlot *slot = table->storeslot;
@@ -5180,6 +5227,34 @@ AfterTriggerFreeQuery(AfterTriggersQueryData *qs)
 	 */
 	qs->tables = NIL;
 	list_free_deep(tables);
+
+	/* Release prolonged tuplestores at the end of the outmost query */
+	if (afterTriggers.query_depth == 0)
+	{
+		foreach(lc, afterTriggers.prolonged_tuplestores)
+		{
+			ts = (Tuplestorestate *) lfirst(lc);
+			if (ts)
+				tuplestore_end(ts);
+		}
+		afterTriggers.prolonged_tuplestores = NIL;
+	}
+}
+
+/*
+ * Release the tuplestore, or append it to the prolonged tuplestores list.
+ */
+static void
+release_or_prolong_tuplestore(Tuplestorestate *ts, bool prolonged)
+{
+	if (prolonged && afterTriggers.query_depth > 0)
+	{
+		MemoryContext oldcxt = MemoryContextSwitchTo(CurTransactionContext);
+		afterTriggers.prolonged_tuplestores = lappend(afterTriggers.prolonged_tuplestores, ts);
+		MemoryContextSwitchTo(oldcxt);
+	}
+	else
+		tuplestore_end(ts);
 }
 
 
diff --git a/src/include/commands/trigger.h b/src/include/commands/trigger.h
index 8a5a9fe642..6718514d34 100644
--- a/src/include/commands/trigger.h
+++ b/src/include/commands/trigger.h
@@ -265,6 +265,8 @@ extern void AfterTriggerEndSubXact(bool isCommit);
 extern void AfterTriggerSetState(ConstraintsSetStmt *stmt);
 extern bool AfterTriggerPendingOnRel(Oid relid);
 
+extern void SetTransitionTablePreserved(Oid relid, CmdType cmdType);
+
 
 /*
  * in utils/adt/ri_triggers.c
-- 
2.25.1


--Multipart=_Tue__2_Jul_2024_17_03_11_+0900_6OrVBZxOB4o_k6A1
Content-Type: text/x-diff;
 name="v33-0002-Add-relisivm-column-to-pg_class-system-catalog.patch"
Content-Disposition: attachment;
 filename="v33-0002-Add-relisivm-column-to-pg_class-system-catalog.patch"
Content-Transfer-Encoding: 7bit



^ permalink  raw  reply  [nested|flat] 249+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
  0 siblings, 0 replies; 249+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 249+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
  0 siblings, 0 replies; 249+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 249+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
  0 siblings, 0 replies; 249+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 249+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
  0 siblings, 0 replies; 249+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 249+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
  0 siblings, 0 replies; 249+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 249+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
  0 siblings, 0 replies; 249+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 249+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
  0 siblings, 0 replies; 249+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 249+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
  0 siblings, 0 replies; 249+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 249+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
  0 siblings, 0 replies; 249+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 249+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
  0 siblings, 0 replies; 249+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 249+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
  0 siblings, 0 replies; 249+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 249+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
  0 siblings, 0 replies; 249+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 249+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
  0 siblings, 0 replies; 249+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 249+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
  0 siblings, 0 replies; 249+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 249+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
  0 siblings, 0 replies; 249+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 249+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
  0 siblings, 0 replies; 249+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 249+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
  0 siblings, 0 replies; 249+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 249+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
  0 siblings, 0 replies; 249+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 249+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
  0 siblings, 0 replies; 249+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 249+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
  0 siblings, 0 replies; 249+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 249+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
  0 siblings, 0 replies; 249+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 249+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
  0 siblings, 0 replies; 249+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 249+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
  0 siblings, 0 replies; 249+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 249+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
  0 siblings, 0 replies; 249+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 249+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
  0 siblings, 0 replies; 249+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 249+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
  0 siblings, 0 replies; 249+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 249+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
  0 siblings, 0 replies; 249+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 249+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
  0 siblings, 0 replies; 249+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 249+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
  0 siblings, 0 replies; 249+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 249+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
  0 siblings, 0 replies; 249+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 249+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
  0 siblings, 0 replies; 249+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 249+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
  0 siblings, 0 replies; 249+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 249+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
  0 siblings, 0 replies; 249+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 249+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
  0 siblings, 0 replies; 249+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 249+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
  0 siblings, 0 replies; 249+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 249+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
  0 siblings, 0 replies; 249+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 249+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
  0 siblings, 0 replies; 249+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 249+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
  0 siblings, 0 replies; 249+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 249+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
  0 siblings, 0 replies; 249+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 249+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
  0 siblings, 0 replies; 249+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 249+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
  0 siblings, 0 replies; 249+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 249+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
  0 siblings, 0 replies; 249+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 249+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
  0 siblings, 0 replies; 249+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 249+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
  0 siblings, 0 replies; 249+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 249+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
  0 siblings, 0 replies; 249+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 249+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
  0 siblings, 0 replies; 249+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 249+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
  0 siblings, 0 replies; 249+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 249+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
  0 siblings, 0 replies; 249+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 249+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
  0 siblings, 0 replies; 249+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 249+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
  0 siblings, 0 replies; 249+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 249+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
  0 siblings, 0 replies; 249+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 249+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
  0 siblings, 0 replies; 249+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 249+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
  0 siblings, 0 replies; 249+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 249+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
  0 siblings, 0 replies; 249+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 249+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
  0 siblings, 0 replies; 249+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 249+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
  0 siblings, 0 replies; 249+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 249+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
  0 siblings, 0 replies; 249+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 249+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
  0 siblings, 0 replies; 249+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 249+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
  0 siblings, 0 replies; 249+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 249+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
  0 siblings, 0 replies; 249+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 249+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
  0 siblings, 0 replies; 249+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 249+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
  0 siblings, 0 replies; 249+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 249+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
  0 siblings, 0 replies; 249+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 249+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
  0 siblings, 0 replies; 249+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 249+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
  0 siblings, 0 replies; 249+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 249+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
  0 siblings, 0 replies; 249+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 249+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
  0 siblings, 0 replies; 249+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 249+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
  0 siblings, 0 replies; 249+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 249+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
  0 siblings, 0 replies; 249+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 249+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
  0 siblings, 0 replies; 249+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 249+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
  0 siblings, 0 replies; 249+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 249+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
  0 siblings, 0 replies; 249+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 249+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
  0 siblings, 0 replies; 249+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 249+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
  0 siblings, 0 replies; 249+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 249+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
  0 siblings, 0 replies; 249+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 249+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
  0 siblings, 0 replies; 249+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 249+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
  0 siblings, 0 replies; 249+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 249+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
  0 siblings, 0 replies; 249+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 249+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
  0 siblings, 0 replies; 249+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 249+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
  0 siblings, 0 replies; 249+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 249+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
  0 siblings, 0 replies; 249+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 249+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
  0 siblings, 0 replies; 249+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 249+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
  0 siblings, 0 replies; 249+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 249+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
  0 siblings, 0 replies; 249+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 249+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
  0 siblings, 0 replies; 249+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 249+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
  0 siblings, 0 replies; 249+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 249+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
  0 siblings, 0 replies; 249+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 249+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
  0 siblings, 0 replies; 249+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 249+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
  0 siblings, 0 replies; 249+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 249+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
  0 siblings, 0 replies; 249+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 249+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
  0 siblings, 0 replies; 249+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 249+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
  0 siblings, 0 replies; 249+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 249+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
  0 siblings, 0 replies; 249+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 249+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
  0 siblings, 0 replies; 249+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 249+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
  0 siblings, 0 replies; 249+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 249+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
  0 siblings, 0 replies; 249+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 249+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
  0 siblings, 0 replies; 249+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 249+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
  0 siblings, 0 replies; 249+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 249+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
  0 siblings, 0 replies; 249+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 249+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
  0 siblings, 0 replies; 249+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 249+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
  0 siblings, 0 replies; 249+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 249+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
  0 siblings, 0 replies; 249+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 249+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
  0 siblings, 0 replies; 249+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 249+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
  0 siblings, 0 replies; 249+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 249+ messages in thread