agora inbox for pgsql-hackers@postgresql.org
help / color / mirror / Atom feed[PATCH v5] contrib/sslinfo: Add ssl_group_info
249+ messages / 2 participants
[nested] [flat]
* [PATCH v5] contrib/sslinfo: Add ssl_group_info
@ 2026-02-19 15:33 Dmitrii Dolgov <9erthalion6@gmail.com>
0 siblings, 0 replies; 249+ messages in thread
From: Dmitrii Dolgov @ 2026-02-19 15:33 UTC (permalink / raw)
Add a new function to sslinfo ssl_group_info to show SSL groups,
including negotiated, supported and shared. It's useful for diagnostic
purposes, to identify what's being used and supported, e.g. which key
share is being negotiated. Few examples, for openssl 3.2.4:
select * from ssl_group_info();
type | name
------------+--------------------
negotiated | X25519MLKEM768
shared | X25519MLKEM768
shared | x25519
supported | X25519MLKEM768
supported | x25519
[...]
The implementation is inspired by ssl_print_groups from openssl.
---
configure | 36 +++++
configure.ac | 4 +
contrib/sslinfo/Makefile | 2 +-
contrib/sslinfo/meson.build | 1 +
contrib/sslinfo/sslinfo--1.2--1.3.sql | 10 ++
contrib/sslinfo/sslinfo.c | 189 +++++++++++++++++++++++++-
contrib/sslinfo/sslinfo.control | 2 +-
doc/src/sgml/sslinfo.sgml | 45 ++++++
meson.build | 5 +
src/include/pg_config.h.in | 11 ++
src/tools/pgindent/typedefs.list | 1 +
11 files changed, 303 insertions(+), 3 deletions(-)
create mode 100644 contrib/sslinfo/sslinfo--1.2--1.3.sql
diff --git a/configure b/configure
index 5aec0afa9ab..b867c188ffc 100755
--- a/configure
+++ b/configure
@@ -13189,6 +13189,42 @@ _ACEOF
fi
done
+ # Functions for groups support, some of them are real functions, some are
+ # just wrappers around SSL_ctrl.
+ for ac_func in SSL_group_to_name
+do :
+ ac_fn_c_check_func "$LINENO" "SSL_group_to_name" "ac_cv_func_SSL_group_to_name"
+if test "x$ac_cv_func_SSL_group_to_name" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_SSL_GROUP_TO_NAME 1
+_ACEOF
+
+fi
+done
+
+ac_fn_c_check_decl "$LINENO" "SSL_get1_groups" "ac_cv_have_decl_SSL_get1_groups" "#include <openssl/ssl.h>
+"
+if test "x$ac_cv_have_decl_SSL_get1_groups" = xyes; then :
+ ac_have_decl=1
+else
+ ac_have_decl=0
+fi
+
+cat >>confdefs.h <<_ACEOF
+#define HAVE_DECL_SSL_GET1_GROUPS $ac_have_decl
+_ACEOF
+ac_fn_c_check_decl "$LINENO" "SSL_get_negotiated_group" "ac_cv_have_decl_SSL_get_negotiated_group" "#include <openssl/ssl.h>
+"
+if test "x$ac_cv_have_decl_SSL_get_negotiated_group" = xyes; then :
+ ac_have_decl=1
+else
+ ac_have_decl=0
+fi
+
+cat >>confdefs.h <<_ACEOF
+#define HAVE_DECL_SSL_GET_NEGOTIATED_GROUP $ac_have_decl
+_ACEOF
+
$as_echo "#define USE_OPENSSL 1" >>confdefs.h
diff --git a/configure.ac b/configure.ac
index fead9a6ce99..6b3bd8667bc 100644
--- a/configure.ac
+++ b/configure.ac
@@ -1444,6 +1444,10 @@ if test "$with_ssl" = openssl ; then
AC_CHECK_FUNCS([SSL_CTX_set_cert_cb])
# Function introduced in OpenSSL 1.1.1, not in LibreSSL.
AC_CHECK_FUNCS([X509_get_signature_info SSL_CTX_set_num_tickets SSL_CTX_set_keylog_callback])
+ # Functions for groups support, some of them are real functions, some are
+ # just wrappers around SSL_ctrl.
+ AC_CHECK_FUNCS([SSL_group_to_name])
+ AC_CHECK_DECLS([SSL_get1_groups, SSL_get_negotiated_group], [], [], [#include <openssl/ssl.h>])
AC_DEFINE([USE_OPENSSL], 1, [Define to 1 to build with OpenSSL support. (--with-ssl=openssl)])
elif test "$with_ssl" != no ; then
AC_MSG_ERROR([--with-ssl must specify openssl])
diff --git a/contrib/sslinfo/Makefile b/contrib/sslinfo/Makefile
index 14305594e2d..d968ef2abfd 100644
--- a/contrib/sslinfo/Makefile
+++ b/contrib/sslinfo/Makefile
@@ -6,7 +6,7 @@ OBJS = \
sslinfo.o
EXTENSION = sslinfo
-DATA = sslinfo--1.2.sql sslinfo--1.1--1.2.sql sslinfo--1.0--1.1.sql
+DATA = sslinfo--1.2.sql sslinfo--1.2--1.3.sql sslinfo--1.1--1.2.sql sslinfo--1.0--1.1.sql
PGFILEDESC = "sslinfo - information about client SSL certificate"
ifdef USE_PGXS
diff --git a/contrib/sslinfo/meson.build b/contrib/sslinfo/meson.build
index 6e9cb96430a..27737562925 100644
--- a/contrib/sslinfo/meson.build
+++ b/contrib/sslinfo/meson.build
@@ -26,6 +26,7 @@ install_data(
'sslinfo--1.0--1.1.sql',
'sslinfo--1.1--1.2.sql',
'sslinfo--1.2.sql',
+ 'sslinfo--1.2--1.3.sql',
'sslinfo.control',
kwargs: contrib_data_args,
)
diff --git a/contrib/sslinfo/sslinfo--1.2--1.3.sql b/contrib/sslinfo/sslinfo--1.2--1.3.sql
new file mode 100644
index 00000000000..40fd0ea2b9c
--- /dev/null
+++ b/contrib/sslinfo/sslinfo--1.2--1.3.sql
@@ -0,0 +1,10 @@
+/* contrib/sslinfo/sslinfo--1.2--1.3.sql */
+
+-- complain if script is sourced in psql, rather than via ALTER EXTENSION
+\echo Use "ALTER EXTENSION sslinfo UPDATE TO '1.3'" to load this file. \quit
+
+CREATE FUNCTION
+ssl_group_info(OUT group_type text, OUT name text
+) RETURNS SETOF record
+AS 'MODULE_PATHNAME', 'ssl_group_info'
+LANGUAGE C STRICT PARALLEL RESTRICTED;
diff --git a/contrib/sslinfo/sslinfo.c b/contrib/sslinfo/sslinfo.c
index 2b9eb90b093..d3f50bc03bf 100644
--- a/contrib/sslinfo/sslinfo.c
+++ b/contrib/sslinfo/sslinfo.c
@@ -19,6 +19,11 @@
#include "miscadmin.h"
#include "utils/builtins.h"
+#define HAVE_SSL_GROUPS \
+ defined(HAVE_DECL_SSL_GET1_GROUPS) && \
+ defined(HAVE_DECL_SSL_GET_NEGOTIATED_GROUP) && \
+ defined(HAVE_SSL_GROUP_TO_NAME)
+
PG_MODULE_MAGIC_EXT(
.name = "sslinfo",
.version = PG_VERSION
@@ -28,13 +33,28 @@ static Datum X509_NAME_field_to_text(X509_NAME *name, text *fieldName);
static Datum ASN1_STRING_to_text(ASN1_STRING *str);
/*
- * Function context for data persisting over repeated calls.
+ * Function context for data persisting over repeated calls of
+ * ssl_extension_info.
*/
typedef struct
{
TupleDesc tupdesc;
} SSLExtensionInfoContext;
+/*
+ * Function context for data persisting over repeated calls of
+ * ssl_group_info.
+ */
+typedef struct
+{
+ TupleDesc tupdesc;
+ int nshared;
+ int nsupported;
+
+ /* Supported groups have to be stored separately */
+ int *supported_groups;
+} SSLGroupInfoContext;
+
/*
* Indicates whether current session uses SSL
*
@@ -474,3 +494,170 @@ ssl_extension_info(PG_FUNCTION_ARGS)
/* All done */
SRF_RETURN_DONE(funcctx);
}
+
+/*
+ * Returns information about TLS groups.
+ *
+ * Returns setof record made of the following values:
+ * - type of the group: negotiated, shared, supported.
+ * - name of the group.
+ */
+PG_FUNCTION_INFO_V1(ssl_group_info);
+Datum
+ssl_group_info(PG_FUNCTION_ARGS)
+{
+#if HAVE_SSL_GROUPS
+ /*
+ * If we lack SSL groups API, we still need to do SRF stuff. Thus the
+ * condition doesn't cover the whole function, but only parts of it. This
+ * particular one is only to avoid unused variable warning, in case if
+ * HAVE_SSL_GROUPS is false.
+ */
+ SSL *ssl = MyProcPort->ssl;
+#endif
+
+ FuncCallContext *funcctx;
+ int call_cntr = 0;
+ int max_calls = 0;
+ MemoryContext oldcontext;
+ SSLGroupInfoContext *fctx;
+
+ if (SRF_IS_FIRSTCALL())
+ {
+
+ TupleDesc tupdesc;
+
+ /* create a function context for cross-call persistence */
+ funcctx = SRF_FIRSTCALL_INIT();
+
+ /*
+ * Switch to memory context appropriate for multiple function calls
+ */
+ oldcontext = MemoryContextSwitchTo(funcctx->multi_call_memory_ctx);
+
+ /* Create a user function context for cross-call persistence */
+ fctx = palloc_object(SSLGroupInfoContext);
+
+ /* Construct tuple descriptor */
+ if (get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_COMPOSITE)
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("function returning record called in context that cannot accept type record")));
+ fctx->tupdesc = BlessTupleDesc(tupdesc);
+
+ if (!MyProcPort->ssl_in_use)
+ {
+ /* fast track when no results */
+ MemoryContextSwitchTo(oldcontext);
+ SRF_RETURN_DONE(funcctx);
+ }
+
+#if HAVE_SSL_GROUPS
+ if (ssl != NULL)
+ {
+ fctx->nsupported = SSL_get1_groups(ssl, NULL);
+ fctx->nshared = SSL_get_shared_group(ssl, -1);
+
+ fctx->supported_groups =
+ palloc(fctx->nsupported * sizeof(*fctx->supported_groups));
+ SSL_get1_groups(ssl, fctx->supported_groups);
+
+ /*
+ * Set max_calls as the number of supported groups plus the number
+ * of shared groups plus one negotiated group.
+ */
+ max_calls = fctx->nsupported + fctx->nshared + 1;
+ }
+
+ if (max_calls > 0)
+ {
+ /* got results, keep track of them */
+ funcctx->max_calls = max_calls;
+ funcctx->user_fctx = fctx;
+ }
+ else
+ {
+ /* fast track when no results */
+ MemoryContextSwitchTo(oldcontext);
+ SRF_RETURN_DONE(funcctx);
+ }
+#else
+ /* SSL groups API is not present, skip */
+ MemoryContextSwitchTo(oldcontext);
+ SRF_RETURN_DONE(funcctx);
+#endif
+
+ MemoryContextSwitchTo(oldcontext);
+ }
+
+ /* stuff done on every call of the function */
+ funcctx = SRF_PERCALL_SETUP();
+
+ /*
+ * Initialize per-call variables.
+ */
+ call_cntr = funcctx->call_cntr;
+ max_calls = funcctx->max_calls;
+ fctx = funcctx->user_fctx;
+
+ /* do while there are more left to send */
+ if (call_cntr < max_calls)
+ {
+#if HAVE_SSL_GROUPS
+ Datum values[2];
+ bool nulls[2];
+ HeapTuple tuple;
+ Datum result,
+ group_type;
+ int nid;
+ const char *group_name;
+
+ /* Send the negotiated group first */
+ if (call_cntr == 0)
+ {
+ nid = SSL_get_negotiated_group(ssl);
+ group_type = CStringGetTextDatum("negotiated");
+ }
+ /* Then the shared groups */
+ else if (call_cntr < fctx->nshared + 1)
+ {
+ nid = SSL_get_shared_group(ssl, call_cntr - 1);
+ group_type = CStringGetTextDatum("shared");
+ }
+ /* And finally the supported groups */
+ else if (call_cntr < fctx->nsupported + fctx->nshared + 1)
+ {
+ nid = fctx->supported_groups[call_cntr - fctx->nshared - 1];
+ group_type = CStringGetTextDatum("supported");
+ }
+ else
+ SRF_RETURN_DONE(funcctx);
+
+ /*
+ * SSL_group_to_name can return NULL in case of an error, e.g. when no
+ * such name was registered for some reason.
+ */
+ group_name = SSL_group_to_name(ssl, nid);
+ if (group_name == NULL)
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("unknown OpenSSL group at position %d",
+ call_cntr)));
+
+ values[0] = group_type;
+ nulls[0] = false;
+
+ values[1] = CStringGetTextDatum(group_name);
+ nulls[1] = false;
+
+ /* Build tuple */
+ tuple = heap_form_tuple(fctx->tupdesc, values, nulls);
+ result = HeapTupleGetDatum(tuple);
+
+ SRF_RETURN_NEXT(funcctx, result);
+#endif
+ }
+
+ /* All done */
+ SRF_RETURN_DONE(funcctx);
+}
diff --git a/contrib/sslinfo/sslinfo.control b/contrib/sslinfo/sslinfo.control
index c7754f924cf..b53e95b7da8 100644
--- a/contrib/sslinfo/sslinfo.control
+++ b/contrib/sslinfo/sslinfo.control
@@ -1,5 +1,5 @@
# sslinfo extension
comment = 'information about SSL certificates'
-default_version = '1.2'
+default_version = '1.3'
module_pathname = '$libdir/sslinfo'
relocatable = true
diff --git a/doc/src/sgml/sslinfo.sgml b/doc/src/sgml/sslinfo.sgml
index 85d49f66537..422745de37c 100644
--- a/doc/src/sgml/sslinfo.sgml
+++ b/doc/src/sgml/sslinfo.sgml
@@ -240,6 +240,51 @@ emailAddress
</para>
</listitem>
</varlistentry>
+
+ <varlistentry>
+ <term>
+ <function>ssl_group_info() returns setof record</function>
+ <indexterm>
+ <primary>ssl_group_info</primary>
+ </indexterm>
+ </term>
+ <listitem>
+ <para>
+ Provide information about TLS groups: group type and group name.
+ The group type value could be one of the following:
+
+ <variablelist>
+ <varlistentry id="ssl-group-info-negotiated">
+ <term><literal>negotiated</literal></term>
+ <listitem>
+ <para>
+ The group used for the handshake key exchange process.
+ </para>
+ </listitem>
+ </varlistentry>
+
+ <varlistentry id="ssl-group-info-shared">
+ <term><literal>shared</literal></term>
+ <listitem>
+ <para>
+ Lisf of named groups shared with the server side.
+ </para>
+ </listitem>
+ </varlistentry>
+
+ <varlistentry id="ssl-group-info-supported">
+ <term><literal>supported</literal></term>
+ <listitem>
+ <para>
+ list of named groups supported by the client for key exchange in the
+ form of "supported_groups" extension.
+ </para>
+ </listitem>
+ </varlistentry>
+ </variablelist>
+ </para>
+ </listitem>
+ </varlistentry>
</variablelist>
</sect2>
diff --git a/meson.build b/meson.build
index 7ee900198db..944db2610b4 100644
--- a/meson.build
+++ b/meson.build
@@ -1684,6 +1684,9 @@ if sslopt in ['auto', 'openssl']
['X509_get_signature_info'],
['SSL_CTX_set_num_tickets'],
['SSL_CTX_set_keylog_callback'],
+
+ # Function for groups support.
+ ['SSL_group_to_name'],
]
are_openssl_funcs_complete = true
@@ -2793,6 +2796,8 @@ decl_checks = [
['strlcpy', 'string.h'],
['strsep', 'string.h'],
['timingsafe_bcmp', 'string.h'],
+ ['SSL_get1_groups', 'openssl/ssl.h'],
+ ['SSL_get_negotiated_group', 'openssl/ssl.h'],
]
# Need to check for function declarations for these functions, because
diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in
index 79379a4d125..a59373ae7ad 100644
--- a/src/include/pg_config.h.in
+++ b/src/include/pg_config.h.in
@@ -101,6 +101,14 @@
don't. */
#undef HAVE_DECL_PWRITEV
+/* Define to 1 if you have the declaration of `SSL_get1_groups', and to 0 if
+ you don't. */
+#undef HAVE_DECL_SSL_GET1_GROUPS
+
+/* Define to 1 if you have the declaration of `SSL_get_negotiated_group', and
+ to 0 if you don't. */
+#undef HAVE_DECL_SSL_GET_NEGOTIATED_GROUP
+
/* Define to 1 if you have the declaration of `strchrnul', and to 0 if you
don't. */
#undef HAVE_DECL_STRCHRNUL
@@ -378,6 +386,9 @@
/* Define to 1 if you have the `SSL_CTX_set_num_tickets' function. */
#undef HAVE_SSL_CTX_SET_NUM_TICKETS
+/* Define to 1 if you have the `SSL_group_to_name' function. */
+#undef HAVE_SSL_GROUP_TO_NAME
+
/* Define to 1 if you have the <stdint.h> header file. */
#undef HAVE_STDINT_H
diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list
index 52f8603a7be..b5ea3c18291 100644
--- a/src/tools/pgindent/typedefs.list
+++ b/src/tools/pgindent/typedefs.list
@@ -2720,6 +2720,7 @@ SQLValueFunction
SQLValueFunctionOp
SSL
SSLExtensionInfoContext
+SSLGroupInfoContext
SSL_CTX
STARTUPINFO
STRLEN
base-commit: e82fc27e095b5a84c578b6e6b43b3396463bd812
--
2.52.0
--zwjbi4ijbttgewfx--
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 249+ messages in thread
From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)
Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.
Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.
For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.
To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).
Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
.github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
# concurrent jobs and retrying older runs have a chance of working.
CLONE_DEPTH: 500
- # At the moment all jobs use 4vcore runners, and none seems to benefit from
- # increasing concurrency further.
- BUILD_JOBS: 4
-
- # It's possible that some jobs benefit from an increased test concurrency,
- # but a default of 4 is a safe bet. Individual jobs can override.
- TEST_JOBS: 4
-
CCACHE_MAXSIZE: "250M"
CCACHE_DIR: ${{ github.workspace }}/ccache_dir
@@ -215,10 +207,19 @@ jobs:
- &nix_sysinfo_step
name: sysinfo
run: |
- id
- uname -a
+ echo -n "Number of cores: "
+ nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+ echo "id: $(id)"
+ echo "uname: $(uname -a)"
+
+ echo ::group::ulimit
ulimit -a -H && ulimit -a -S
+ echo ::endgroup::
+
+ echo ::group::env
env
+ echo ::endgroup::
- name: Parse ci-os-only
id: os
@@ -352,7 +353,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: &ninja_build_cmd |
- ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build ${{env.MBUILD_TARGET}}
ninja -C build -t missingdeps
# Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
echo ::endgroup::
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+ meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
- &linux_collect_cores_step
name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
- name: Build
shell: *su_postgres_shell
run: |
- make -s -j${BUILD_JOBS} world-bin
+ make -s -j$(nproc) world-bin
- *ccache_decide_save_step
- *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
- name: Test world
shell: *su_postgres_shell
run: |
- make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
- *linux_collect_cores_step
- *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
# Run the tests supporting running against an already running
- meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+ meson test ${{env.MTEST_ARGS}} --setup running
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
@@ -893,9 +894,19 @@ jobs:
- name: Sysinfo
run: |
+ echo Number of cores:
+ bash -c nproc
+
+ echo codepage:
chcp
+
+ echo ::group::systeminfo
systeminfo
+ echo ::endgroup::
+
+ echo ::group::env
set
+ echo ::endgroup::
# The TAP tests build an initdb template under build/tmp_install and
# then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
meson setup -Dwerror=true $CONF build-meson
echo "::endgroup::"
- ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+ ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
# gcc, cassert on, dtrace off
- name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
echo "::endgroup::"
- make -s -j${{env.BUILD_JOBS}} clean
- make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+ make -s -j$(nproc) clean
+ make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
headerscheck cpluspluscheck \
EXTRAFLAGS='-fmax-errors=10'
--
2.54.0.450.g9ac3f193c0
--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13a-0008-ci-Make-runs-on-overridable.patch"
^ permalink raw reply [nested|flat] 249+ messages in thread
end of thread, other threads:[~2026-06-10 18:49 UTC | newest]
Thread overview: 249+ messages (download: mbox mbox.gz follow: Atom feed)
-- links below jump to the message on this page --
2026-02-19 15:33 [PATCH v5] contrib/sslinfo: Add ssl_group_info Dmitrii Dolgov <9erthalion6@gmail.com>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
2026-06-10 18:49 [PATCH v13a 7/9] ci: get build/test concurrency from environment Andres Freund <andres@anarazel.de>
This inbox is served by agora; see mirroring instructions
for how to clone and mirror all data and code used for this inbox